You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/06/16 21:44:03 UTC
[01/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Repository: hbase-site
Updated Branches:
refs/heads/asf-site 3aa0489ce -> a7f7dba1b
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
index 38b1859..ee05a1d 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
@@ -398,8 +398,8 @@
<span class="sourceLineNo">390</span> new Entry(new WALKeyImpl(encoded,<a name="line.390"></a>
<span class="sourceLineNo">391</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.391"></a>
<span class="sourceLineNo">392</span> new WALEdit());<a name="line.392"></a>
-<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,<a name="line.393"></a>
-<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT);<a name="line.394"></a>
+<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.393"></a>
+<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT, conf);<a name="line.394"></a>
<span class="sourceLineNo">395</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.395"></a>
<span class="sourceLineNo">396</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.396"></a>
<span class="sourceLineNo">397</span> }<a name="line.397"></a>
@@ -424,8 +424,8 @@
<span class="sourceLineNo">416</span> assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());<a name="line.416"></a>
<span class="sourceLineNo">417</span> fs.createNewFile(parent); // create a recovered.edits file<a name="line.417"></a>
<span class="sourceLineNo">418</span><a name="line.418"></a>
-<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,<a name="line.419"></a>
-<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT);<a name="line.420"></a>
+<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.419"></a>
+<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT, conf);<a name="line.420"></a>
<span class="sourceLineNo">421</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.421"></a>
<span class="sourceLineNo">422</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.422"></a>
<span class="sourceLineNo">423</span> WALFactory.createRecoveredEditsWriter(fs, p, conf).close();<a name="line.423"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.ZombieLastLogWriterRegionServer.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.ZombieLastLogWriterRegionServer.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.ZombieLastLogWriterRegionServer.html
index 38b1859..ee05a1d 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.ZombieLastLogWriterRegionServer.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.ZombieLastLogWriterRegionServer.html
@@ -398,8 +398,8 @@
<span class="sourceLineNo">390</span> new Entry(new WALKeyImpl(encoded,<a name="line.390"></a>
<span class="sourceLineNo">391</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.391"></a>
<span class="sourceLineNo">392</span> new WALEdit());<a name="line.392"></a>
-<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,<a name="line.393"></a>
-<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT);<a name="line.394"></a>
+<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.393"></a>
+<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT, conf);<a name="line.394"></a>
<span class="sourceLineNo">395</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.395"></a>
<span class="sourceLineNo">396</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.396"></a>
<span class="sourceLineNo">397</span> }<a name="line.397"></a>
@@ -424,8 +424,8 @@
<span class="sourceLineNo">416</span> assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());<a name="line.416"></a>
<span class="sourceLineNo">417</span> fs.createNewFile(parent); // create a recovered.edits file<a name="line.417"></a>
<span class="sourceLineNo">418</span><a name="line.418"></a>
-<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,<a name="line.419"></a>
-<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT);<a name="line.420"></a>
+<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.419"></a>
+<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT, conf);<a name="line.420"></a>
<span class="sourceLineNo">421</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.421"></a>
<span class="sourceLineNo">422</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.422"></a>
<span class="sourceLineNo">423</span> WALFactory.createRecoveredEditsWriter(fs, p, conf).close();<a name="line.423"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.html
index 38b1859..ee05a1d 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.html
@@ -398,8 +398,8 @@
<span class="sourceLineNo">390</span> new Entry(new WALKeyImpl(encoded,<a name="line.390"></a>
<span class="sourceLineNo">391</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.391"></a>
<span class="sourceLineNo">392</span> new WALEdit());<a name="line.392"></a>
-<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,<a name="line.393"></a>
-<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT);<a name="line.394"></a>
+<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.393"></a>
+<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT, conf);<a name="line.394"></a>
<span class="sourceLineNo">395</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.395"></a>
<span class="sourceLineNo">396</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.396"></a>
<span class="sourceLineNo">397</span> }<a name="line.397"></a>
@@ -424,8 +424,8 @@
<span class="sourceLineNo">416</span> assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());<a name="line.416"></a>
<span class="sourceLineNo">417</span> fs.createNewFile(parent); // create a recovered.edits file<a name="line.417"></a>
<span class="sourceLineNo">418</span><a name="line.418"></a>
-<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR,<a name="line.419"></a>
-<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT);<a name="line.420"></a>
+<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.419"></a>
+<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT, conf);<a name="line.420"></a>
<span class="sourceLineNo">421</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.421"></a>
<span class="sourceLineNo">422</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.422"></a>
<span class="sourceLineNo">423</span> WALFactory.createRecoveredEditsWriter(fs, p, conf).close();<a name="line.423"></a>
[22/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
Published site at ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/a7f7dba1
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/a7f7dba1
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/a7f7dba1
Branch: refs/heads/asf-site
Commit: a7f7dba1b72a521da9894f72d2b5a70ea7b099e3
Parents: 3aa0489
Author: jenkins <bu...@apache.org>
Authored: Sat Jun 16 21:43:49 2018 +0000
Committer: jenkins <bu...@apache.org>
Committed: Sat Jun 16 21:43:49 2018 +0000
----------------------------------------------------------------------
acid-semantics.html | 4 +-
apache_hbase_reference_guide.pdf | 33976 +++++++++++++----
book.html | 579 +-
bulk-loads.html | 4 +-
checkstyle-aggregate.html | 206 +-
checkstyle.rss | 4 +-
coc.html | 4 +-
dependencies.html | 4 +-
dependency-convergence.html | 4 +-
dependency-info.html | 4 +-
dependency-management.html | 4 +-
devapidocs/constant-values.html | 6 +-
devapidocs/index-all.html | 10 +-
.../class-use/SplitLogWorkerCoordination.html | 4 +-
.../regionserver/class-use/LastSequenceId.html | 4 +-
.../util/class-use/CancelableProgressable.html | 2 +-
.../apache/hadoop/hbase/wal/FSHLogProvider.html | 4 +-
...tter.BoundedLogWriterCreationOutputSink.html | 20 +-
.../WALSplitter.CorruptedLogFileException.html | 6 +-
.../hbase/wal/WALSplitter.EntryBuffers.html | 28 +-
...WALSplitter.LogRecoveredEditsOutputSink.html | 48 +-
.../hbase/wal/WALSplitter.MutationReplay.html | 18 +-
.../hbase/wal/WALSplitter.OutputSink.html | 52 +-
.../wal/WALSplitter.PipelineController.html | 12 +-
.../wal/WALSplitter.RegionEntryBuffer.html | 24 +-
.../hbase/wal/WALSplitter.SinkWriter.html | 16 +-
.../hbase/wal/WALSplitter.WriterAndPath.html | 10 +-
.../hbase/wal/WALSplitter.WriterThread.html | 20 +-
.../apache/hadoop/hbase/wal/WALSplitter.html | 99 +-
.../hadoop/hbase/wal/class-use/WAL.Entry.html | 12 +-
.../hadoop/hbase/wal/class-use/WALFactory.html | 4 +-
.../class-use/WALSplitter.WriterAndPath.html | 8 +-
.../org/apache/hadoop/hbase/Version.html | 6 +-
.../hadoop/hbase/wal/FSHLogProvider.Writer.html | 71 +-
.../apache/hadoop/hbase/wal/FSHLogProvider.html | 71 +-
...tter.BoundedLogWriterCreationOutputSink.html | 2849 +-
.../WALSplitter.CorruptedLogFileException.html | 2849 +-
.../hbase/wal/WALSplitter.EntryBuffers.html | 2849 +-
...WALSplitter.LogRecoveredEditsOutputSink.html | 2849 +-
.../hbase/wal/WALSplitter.MutationReplay.html | 2849 +-
.../hbase/wal/WALSplitter.OutputSink.html | 2849 +-
.../wal/WALSplitter.PipelineController.html | 2849 +-
.../wal/WALSplitter.RegionEntryBuffer.html | 2849 +-
.../hbase/wal/WALSplitter.SinkWriter.html | 2849 +-
.../hbase/wal/WALSplitter.WriterAndPath.html | 2849 +-
.../hbase/wal/WALSplitter.WriterThread.html | 2849 +-
.../apache/hadoop/hbase/wal/WALSplitter.html | 2849 +-
downloads.html | 4 +-
export_control.html | 4 +-
index.html | 4 +-
integration.html | 4 +-
issue-tracking.html | 4 +-
license.html | 4 +-
mail-lists.html | 4 +-
metrics.html | 4 +-
old_news.html | 4 +-
plugin-management.html | 4 +-
plugins.html | 4 +-
poweredbyhbase.html | 4 +-
project-info.html | 4 +-
project-reports.html | 4 +-
project-summary.html | 4 +-
pseudo-distributed.html | 4 +-
replication.html | 4 +-
resources.html | 4 +-
source-repository.html | 4 +-
sponsors.html | 4 +-
supportingprojects.html | 4 +-
team-list.html | 4 +-
.../TestWALFactory.DumbWALActionsListener.html | 10 +-
.../apache/hadoop/hbase/wal/TestWALFactory.html | 54 +-
.../TestWALFactory.DumbWALActionsListener.html | 1230 +-
.../apache/hadoop/hbase/wal/TestWALFactory.html | 1230 +-
.../hbase/wal/TestWALSplit.Corruptions.html | 8 +-
...ALSplit.ZombieLastLogWriterRegionServer.html | 8 +-
.../apache/hadoop/hbase/wal/TestWALSplit.html | 8 +-
76 files changed, 44929 insertions(+), 27243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/acid-semantics.html
----------------------------------------------------------------------
diff --git a/acid-semantics.html b/acid-semantics.html
index bd0b2d5..de49b0c 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Apache HBase (TM) ACID Properties
@@ -601,7 +601,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
[13/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists
<TRUNCATED>
[11/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span> }<a name="line.68
<TRUNCATED>
[12/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span>
<TRUNCATED>
[21/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/apache_hbase_reference_guide.pdf
----------------------------------------------------------------------
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index fe2b224..4bd2008 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,16 +5,16 @@
/Author (Apache HBase Team)
/Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2)
/Producer (Apache HBase Team)
-/ModDate (D:20180615212614+00'00')
-/CreationDate (D:20180615214255+00'00')
+/ModDate (D:20180616212517+00'00')
+/CreationDate (D:20180616214046+00'00')
>>
endobj
2 0 obj
<< /Type /Catalog
/Pages 3 0 R
/Names 26 0 R
-/Outlines 4656 0 R
-/PageLabels 4883 0 R
+/Outlines 4782 0 R
+/PageLabels 5027 0 R
/PageMode /UseOutlines
/OpenAction [7 0 R /FitH 842.89]
/ViewerPreferences << /DisplayDocTitle true
@@ -23,8 +23,8 @@ endobj
endobj
3 0 obj
<< /Type /Pages
-/Count 727
-/Kids [7 0 R 12 0 R 14 0 R 16 0 R 18 0 R 20 0 R 22 0 R 24 0 R 44 0 R 47 0 R 50 0 R 54 0 R 61 0 R 63 0 R 67 0 R 69 0 R 71 0 R 78 0 R 81 0 R 83 0 R 89 0 R 92 0 R 94 0 R 96 0 R 103 0 R 110 0 R 115 0 R 117 0 R 133 0 R 138 0 R 146 0 R 155 0 R 163 0 R 172 0 R 183 0 R 187 0 R 189 0 R 193 0 R 202 0 R 211 0 R 219 0 R 228 0 R 233 0 R 242 0 R 250 0 R 259 0 R 272 0 R 279 0 R 289 0 R 297 0 R 305 0 R 312 0 R 320 0 R 327 0 R 333 0 R 340 0 R 348 0 R 357 0 R 366 0 R 380 0 R 387 0 R 395 0 R 402 0 R 410 0 R 419 0 R 429 0 R 437 0 R 444 0 R 453 0 R 465 0 R 475 0 R 482 0 R 489 0 R 497 0 R 506 0 R 514 0 R 519 0 R 523 0 R 528 0 R 532 0 R 548 0 R 559 0 R 563 0 R 578 0 R 583 0 R 588 0 R 590 0 R 592 0 R 595 0 R 597 0 R 599 0 R 607 0 R 613 0 R 616 0 R 620 0 R 629 0 R 640 0 R 648 0 R 652 0 R 656 0 R 658 0 R 668 0 R 683 0 R 690 0 R 701 0 R 711 0 R 722 0 R 734 0 R 754 0 R 763 0 R 767 0 R 773 0 R 776 0 R 780 0 R 784 0 R 787 0 R 790 0 R 792 0 R 795 0 R 799 0 R 801 0 R 805 0 R 811 0 R 816 0 R 820 0 R 823 0 R 829 0 R
831 0 R 835 0 R 843 0 R 845 0 R 848 0 R 851 0 R 854 0 R 857 0 R 871 0 R 879 0 R 890 0 R 901 0 R 907 0 R 917 0 R 928 0 R 931 0 R 935 0 R 938 0 R 943 0 R 952 0 R 960 0 R 964 0 R 968 0 R 973 0 R 977 0 R 979 0 R 995 0 R 1006 0 R 1011 0 R 1018 0 R 1021 0 R 1029 0 R 1037 0 R 1042 0 R 1047 0 R 1052 0 R 1054 0 R 1056 0 R 1058 0 R 1068 0 R 1076 0 R 1080 0 R 1087 0 R 1094 0 R 1102 0 R 1106 0 R 1112 0 R 1117 0 R 1125 0 R 1129 0 R 1134 0 R 1136 0 R 1142 0 R 1150 0 R 1156 0 R 1163 0 R 1174 0 R 1178 0 R 1180 0 R 1182 0 R 1186 0 R 1189 0 R 1194 0 R 1197 0 R 1209 0 R 1213 0 R 1219 0 R 1227 0 R 1232 0 R 1236 0 R 1240 0 R 1242 0 R 1245 0 R 1248 0 R 1251 0 R 1255 0 R 1259 0 R 1263 0 R 1268 0 R 1272 0 R 1275 0 R 1277 0 R 1287 0 R 1290 0 R 1298 0 R 1307 0 R 1313 0 R 1317 0 R 1319 0 R 1330 0 R 1333 0 R 1339 0 R 1347 0 R 1350 0 R 1357 0 R 1364 0 R 1367 0 R 1369 0 R 1378 0 R 1380 0 R 1382 0 R 1385 0 R 1387 0 R 1389 0 R 1391 0 R 1393 0 R 1396 0 R 1400 0 R 1405 0 R 1407 0 R 1409 0 R 1411 0 R 1416 0 R 1423 0
R 1429 0 R 1432 0 R 1434 0 R 1437 0 R 1441 0 R 1445 0 R 1448 0 R 1450 0 R 1452 0 R 1455 0 R 1460 0 R 1466 0 R 1474 0 R 1488 0 R 1502 0 R 1505 0 R 1510 0 R 1523 0 R 1528 0 R 1543 0 R 1551 0 R 1555 0 R 1564 0 R 1579 0 R 1593 0 R 1601 0 R 1606 0 R 1617 0 R 1622 0 R 1628 0 R 1634 0 R 1646 0 R 1649 0 R 1658 0 R 1661 0 R 1670 0 R 1676 0 R 1680 0 R 1692 0 R 1697 0 R 1703 0 R 1705 0 R 1712 0 R 1720 0 R 1728 0 R 1732 0 R 1734 0 R 1736 0 R 1748 0 R 1754 0 R 1763 0 R 1769 0 R 1782 0 R 1788 0 R 1794 0 R 1805 0 R 1811 0 R 1816 0 R 1821 0 R 1824 0 R 1827 0 R 1832 0 R 1837 0 R 1844 0 R 1848 0 R 1853 0 R 1862 0 R 1867 0 R 1872 0 R 1874 0 R 1883 0 R 1890 0 R 1896 0 R 1901 0 R 1905 0 R 1909 0 R 1914 0 R 1919 0 R 1925 0 R 1927 0 R 1929 0 R 1932 0 R 1943 0 R 1946 0 R 1953 0 R 1961 0 R 1966 0 R 1970 0 R 1975 0 R 1977 0 R 1980 0 R 1985 0 R 1988 0 R 1990 0 R 1993 0 R 1996 0 R 1999 0 R 2009 0 R 2014 0 R 2019 0 R 2021 0 R 2029 0 R 2036 0 R 2043 0 R 2049 0 R 2054 0 R 2056 0 R 2065 0 R 2075 0 R 2085 0 R 2091
0 R 2098 0 R 2100 0 R 2105 0 R 2107 0 R 2109 0 R 2113 0 R 2116 0 R 2119 0 R 2124 0 R 2128 0 R 2139 0 R 2142 0 R 2147 0 R 2150 0 R 2152 0 R 2157 0 R 2167 0 R 2169 0 R 2171 0 R 2173 0 R 2175 0 R 2178 0 R 2180 0 R 2182 0 R 2185 0 R 2187 0 R 2189 0 R 2193 0 R 2198 0 R 2207 0 R 2209 0 R 2211 0 R 2217 0 R 2219 0 R 2224 0 R 2226 0 R 2228 0 R 2235 0 R 2240 0 R 2244 0 R 2249 0 R 2253 0 R 2255 0 R 2257 0 R 2261 0 R 2264 0 R 2266 0 R 2268 0 R 2272 0 R 2274 0 R 2277 0 R 2279 0 R 2281 0 R 2283 0 R 2290 0 R 2293 0 R 2298 0 R 2300 0 R 2302 0 R 2304 0 R 2306 0 R 2314 0 R 2325 0 R 2339 0 R 2350 0 R 2354 0 R 2359 0 R 2363 0 R 2366 0 R 2371 0 R 2377 0 R 2379 0 R 2382 0 R 2384 0 R 2386 0 R 2388 0 R 2393 0 R 2395 0 R 2408 0 R 2411 0 R 2419 0 R 2425 0 R 2437 0 R 2451 0 R 2464 0 R 2483 0 R 2485 0 R 2487 0 R 2491 0 R 2509 0 R 2515 0 R 2527 0 R 2531 0 R 2535 0 R 2544 0 R 2556 0 R 2561 0 R 2571 0 R 2584 0 R 2603 0 R 2612 0 R 2615 0 R 2624 0 R 2641 0 R 2648 0 R 2651 0 R 2656 0 R 2660 0 R 2663 0 R 2672 0 R 26
81 0 R 2684 0 R 2686 0 R 2690 0 R 2704 0 R 2713 0 R 2718 0 R 2722 0 R 2725 0 R 2727 0 R 2729 0 R 2731 0 R 2736 0 R 2749 0 R 2759 0 R 2767 0 R 2773 0 R 2778 0 R 2789 0 R 2796 0 R 2802 0 R 2804 0 R 2813 0 R 2821 0 R 2823 0 R 2831 0 R 2839 0 R 2841 0 R 2850 0 R 2853 0 R 2863 0 R 2867 0 R 2876 0 R 2884 0 R 2889 0 R 2893 0 R 2897 0 R 2899 0 R 2905 0 R 2909 0 R 2913 0 R 2919 0 R 2925 0 R 2928 0 R 2934 0 R 2938 0 R 2947 0 R 2952 0 R 2958 0 R 2968 0 R 2974 0 R 2981 0 R 2984 0 R 2987 0 R 2994 0 R 2999 0 R 3002 0 R 3007 0 R 3017 0 R 3022 0 R 3024 0 R 3028 0 R 3035 0 R 3038 0 R 3049 0 R 3052 0 R 3059 0 R 3067 0 R 3074 0 R 3079 0 R 3088 0 R 3093 0 R 3097 0 R 3105 0 R 3110 0 R 3114 0 R 3116 0 R 3127 0 R 3132 0 R 3135 0 R 3137 0 R 3139 0 R 3149 0 R 3156 0 R 3160 0 R 3163 0 R 3169 0 R 3173 0 R 3176 0 R 3180 0 R 3185 0 R 3193 0 R 3198 0 R 3203 0 R 3208 0 R 3210 0 R 3213 0 R 3215 0 R 3219 0 R 3230 0 R 3232 0 R 3236 0 R 3239 0 R 3243 0 R 3246 0 R 3250 0 R 3252 0 R 3265 0 R 3270 0 R 3275 0 R 3281 0 R
3289 0 R 3291 0 R 3299 0 R 3317 0 R 3329 0 R 3335 0 R 3351 0 R 3354 0 R 3359 0 R 3361 0 R 3368 0 R 3371 0 R 3376 0 R 3379 0 R 3381 0 R 3383 0 R 3385 0 R 3388 0 R 3406 0 R 3409 0 R 3414 0 R 3420 0 R 3430 0 R 3435 0 R 3445 0 R 3455 0 R 3463 0 R 3468 0 R 3474 0 R 3479 0 R 3482 0 R 3490 0 R 3494 0 R 3499 0 R 3504 0 R 3517 0 R 3520 0 R 3526 0 R 3532 0 R 3537 0 R 3546 0 R 3555 0 R 3561 0 R 3570 0 R 3579 0 R 3584 0 R 3590 0 R 3596 0 R 3600 0 R 3602 0 R 3608 0 R 3615 0 R 3617 0 R 3623 0 R 3625 0 R 3631 0 R 3639 0 R 3645 0 R 3654 0 R 3661 0 R 3672 0 R 3682 0 R 3694 0 R 3707 0 R 3710 0 R 3712 0 R 3716 0 R 3729 0 R 3735 0 R 3740 0 R 3746 0 R 3750 0 R 3753 0 R 3758 0 R 3760 0 R 3764 0 R 3766 0 R 3770 0 R 3773 0 R 3776 0 R 3784 0 R 3786 0 R 3790 0 R 3793 0 R 3801 0 R 3809 0 R 3813 0 R 3816 0 R 3818 0 R 3822 0 R 3827 0 R 3832 0 R 3835 0 R 3844 0 R 3849 0 R 3853 0 R 3856 0 R 3864 0 R 3869 0 R 3877 0 R 3882 0 R 3884 0 R 3890 0 R 3892 0 R 3897 0 R 3901 0 R 3907 0 R 3911 0 R 3923 0 R 3939 0 R 3954 0
R 3959 0 R 3962 0 R 3965 0 R 3971 0 R 3976 0 R 3978 0 R 3980 0 R 3982 0 R 3984 0 R 3986 0 R 3995 0 R 3999 0 R 4003 0 R 4007 0 R 4009 0 R 4016 0 R 4026 0 R 4035 0 R 4038 0 R 4041 0 R 4043 0 R 4050 0 R 4057 0 R 4067 0 R 4071 0 R 4074 0 R 4078 0 R 4082 0 R 4088 0 R 4091 0 R 4106 0 R 4111 0 R 4134 0 R 4138 0 R 4145 0 R 4156 0 R 4165 0 R 4168 0 R 4171 0 R 4174 0 R 4190 0 R 4195 0 R 4202 0 R]
+/Count 748
+/Kids [7 0 R 12 0 R 14 0 R 16 0 R 18 0 R 20 0 R 22 0 R 24 0 R 44 0 R 47 0 R 50 0 R 54 0 R 61 0 R 63 0 R 67 0 R 69 0 R 71 0 R 78 0 R 81 0 R 83 0 R 89 0 R 92 0 R 94 0 R 96 0 R 103 0 R 110 0 R 115 0 R 117 0 R 133 0 R 138 0 R 146 0 R 155 0 R 163 0 R 172 0 R 183 0 R 187 0 R 189 0 R 193 0 R 202 0 R 211 0 R 219 0 R 228 0 R 233 0 R 242 0 R 250 0 R 259 0 R 272 0 R 279 0 R 289 0 R 297 0 R 305 0 R 312 0 R 320 0 R 327 0 R 333 0 R 340 0 R 348 0 R 357 0 R 366 0 R 380 0 R 387 0 R 395 0 R 402 0 R 410 0 R 419 0 R 429 0 R 437 0 R 444 0 R 453 0 R 465 0 R 475 0 R 482 0 R 489 0 R 497 0 R 506 0 R 514 0 R 519 0 R 523 0 R 528 0 R 532 0 R 548 0 R 559 0 R 563 0 R 578 0 R 583 0 R 588 0 R 590 0 R 592 0 R 595 0 R 597 0 R 599 0 R 607 0 R 613 0 R 616 0 R 620 0 R 629 0 R 640 0 R 648 0 R 652 0 R 656 0 R 658 0 R 668 0 R 683 0 R 690 0 R 701 0 R 711 0 R 722 0 R 734 0 R 754 0 R 763 0 R 767 0 R 773 0 R 776 0 R 780 0 R 784 0 R 787 0 R 790 0 R 792 0 R 795 0 R 799 0 R 801 0 R 805 0 R 811 0 R 816 0 R 820 0 R 823 0 R 829 0 R
831 0 R 835 0 R 843 0 R 845 0 R 848 0 R 851 0 R 854 0 R 857 0 R 871 0 R 879 0 R 890 0 R 901 0 R 907 0 R 917 0 R 928 0 R 931 0 R 935 0 R 938 0 R 943 0 R 952 0 R 960 0 R 964 0 R 968 0 R 973 0 R 977 0 R 979 0 R 995 0 R 1006 0 R 1011 0 R 1018 0 R 1021 0 R 1029 0 R 1037 0 R 1042 0 R 1047 0 R 1052 0 R 1054 0 R 1056 0 R 1058 0 R 1068 0 R 1076 0 R 1080 0 R 1087 0 R 1094 0 R 1102 0 R 1106 0 R 1112 0 R 1117 0 R 1125 0 R 1129 0 R 1134 0 R 1136 0 R 1142 0 R 1150 0 R 1156 0 R 1163 0 R 1174 0 R 1178 0 R 1180 0 R 1182 0 R 1186 0 R 1189 0 R 1194 0 R 1197 0 R 1209 0 R 1213 0 R 1219 0 R 1227 0 R 1232 0 R 1236 0 R 1240 0 R 1242 0 R 1245 0 R 1248 0 R 1251 0 R 1255 0 R 1259 0 R 1263 0 R 1268 0 R 1272 0 R 1275 0 R 1277 0 R 1287 0 R 1290 0 R 1298 0 R 1307 0 R 1313 0 R 1317 0 R 1319 0 R 1330 0 R 1333 0 R 1339 0 R 1347 0 R 1350 0 R 1357 0 R 1364 0 R 1367 0 R 1369 0 R 1378 0 R 1380 0 R 1382 0 R 1385 0 R 1387 0 R 1389 0 R 1391 0 R 1393 0 R 1396 0 R 1400 0 R 1405 0 R 1407 0 R 1409 0 R 1411 0 R 1416 0 R 1423 0
R 1429 0 R 1432 0 R 1434 0 R 1437 0 R 1441 0 R 1445 0 R 1448 0 R 1450 0 R 1452 0 R 1455 0 R 1460 0 R 1466 0 R 1474 0 R 1488 0 R 1502 0 R 1505 0 R 1510 0 R 1523 0 R 1528 0 R 1543 0 R 1551 0 R 1555 0 R 1564 0 R 1579 0 R 1593 0 R 1601 0 R 1606 0 R 1617 0 R 1622 0 R 1628 0 R 1634 0 R 1646 0 R 1649 0 R 1658 0 R 1661 0 R 1670 0 R 1676 0 R 1680 0 R 1692 0 R 1697 0 R 1703 0 R 1705 0 R 1712 0 R 1720 0 R 1728 0 R 1732 0 R 1734 0 R 1736 0 R 1748 0 R 1754 0 R 1763 0 R 1769 0 R 1782 0 R 1788 0 R 1794 0 R 1805 0 R 1811 0 R 1816 0 R 1821 0 R 1824 0 R 1827 0 R 1832 0 R 1837 0 R 1844 0 R 1848 0 R 1853 0 R 1862 0 R 1867 0 R 1872 0 R 1874 0 R 1883 0 R 1890 0 R 1896 0 R 1901 0 R 1905 0 R 1909 0 R 1914 0 R 1919 0 R 1925 0 R 1927 0 R 1929 0 R 1932 0 R 1943 0 R 1946 0 R 1953 0 R 1961 0 R 1966 0 R 1970 0 R 1975 0 R 1977 0 R 1980 0 R 1985 0 R 1988 0 R 1990 0 R 1993 0 R 1996 0 R 1999 0 R 2009 0 R 2014 0 R 2019 0 R 2021 0 R 2029 0 R 2036 0 R 2043 0 R 2049 0 R 2054 0 R 2056 0 R 2065 0 R 2075 0 R 2085 0 R 2091
0 R 2098 0 R 2100 0 R 2105 0 R 2107 0 R 2109 0 R 2113 0 R 2116 0 R 2119 0 R 2124 0 R 2128 0 R 2139 0 R 2142 0 R 2147 0 R 2150 0 R 2152 0 R 2157 0 R 2167 0 R 2169 0 R 2171 0 R 2173 0 R 2175 0 R 2178 0 R 2180 0 R 2182 0 R 2185 0 R 2187 0 R 2189 0 R 2193 0 R 2198 0 R 2207 0 R 2209 0 R 2211 0 R 2217 0 R 2219 0 R 2224 0 R 2226 0 R 2228 0 R 2235 0 R 2240 0 R 2244 0 R 2249 0 R 2253 0 R 2255 0 R 2257 0 R 2261 0 R 2264 0 R 2266 0 R 2268 0 R 2272 0 R 2274 0 R 2277 0 R 2279 0 R 2281 0 R 2283 0 R 2290 0 R 2293 0 R 2298 0 R 2300 0 R 2302 0 R 2304 0 R 2306 0 R 2314 0 R 2325 0 R 2339 0 R 2350 0 R 2354 0 R 2359 0 R 2363 0 R 2366 0 R 2371 0 R 2377 0 R 2379 0 R 2382 0 R 2384 0 R 2386 0 R 2388 0 R 2393 0 R 2395 0 R 2408 0 R 2411 0 R 2419 0 R 2425 0 R 2437 0 R 2451 0 R 2464 0 R 2483 0 R 2485 0 R 2487 0 R 2491 0 R 2509 0 R 2515 0 R 2527 0 R 2531 0 R 2535 0 R 2544 0 R 2556 0 R 2561 0 R 2571 0 R 2584 0 R 2603 0 R 2612 0 R 2615 0 R 2624 0 R 2641 0 R 2648 0 R 2651 0 R 2656 0 R 2660 0 R 2663 0 R 2672 0 R 26
81 0 R 2684 0 R 2686 0 R 2690 0 R 2704 0 R 2713 0 R 2718 0 R 2722 0 R 2725 0 R 2727 0 R 2729 0 R 2731 0 R 2736 0 R 2749 0 R 2759 0 R 2767 0 R 2773 0 R 2778 0 R 2789 0 R 2796 0 R 2802 0 R 2804 0 R 2813 0 R 2821 0 R 2823 0 R 2831 0 R 2839 0 R 2841 0 R 2850 0 R 2853 0 R 2863 0 R 2867 0 R 2876 0 R 2884 0 R 2889 0 R 2893 0 R 2897 0 R 2899 0 R 2905 0 R 2909 0 R 2913 0 R 2919 0 R 2925 0 R 2928 0 R 2934 0 R 2938 0 R 2947 0 R 2952 0 R 2958 0 R 2968 0 R 2974 0 R 2981 0 R 2984 0 R 2987 0 R 2994 0 R 2999 0 R 3002 0 R 3007 0 R 3017 0 R 3022 0 R 3024 0 R 3028 0 R 3035 0 R 3038 0 R 3049 0 R 3052 0 R 3059 0 R 3067 0 R 3074 0 R 3079 0 R 3088 0 R 3093 0 R 3097 0 R 3105 0 R 3110 0 R 3114 0 R 3116 0 R 3127 0 R 3132 0 R 3135 0 R 3137 0 R 3139 0 R 3149 0 R 3156 0 R 3160 0 R 3163 0 R 3169 0 R 3173 0 R 3176 0 R 3180 0 R 3185 0 R 3193 0 R 3198 0 R 3203 0 R 3208 0 R 3210 0 R 3213 0 R 3215 0 R 3219 0 R 3230 0 R 3232 0 R 3236 0 R 3239 0 R 3243 0 R 3246 0 R 3250 0 R 3252 0 R 3265 0 R 3270 0 R 3275 0 R 3281 0 R
3289 0 R 3291 0 R 3299 0 R 3317 0 R 3329 0 R 3335 0 R 3351 0 R 3354 0 R 3359 0 R 3361 0 R 3368 0 R 3371 0 R 3376 0 R 3379 0 R 3381 0 R 3383 0 R 3385 0 R 3388 0 R 3406 0 R 3409 0 R 3414 0 R 3420 0 R 3430 0 R 3435 0 R 3445 0 R 3455 0 R 3463 0 R 3468 0 R 3474 0 R 3479 0 R 3482 0 R 3490 0 R 3494 0 R 3499 0 R 3504 0 R 3517 0 R 3520 0 R 3526 0 R 3532 0 R 3537 0 R 3546 0 R 3555 0 R 3561 0 R 3570 0 R 3579 0 R 3584 0 R 3590 0 R 3596 0 R 3600 0 R 3602 0 R 3608 0 R 3615 0 R 3617 0 R 3623 0 R 3625 0 R 3631 0 R 3639 0 R 3645 0 R 3654 0 R 3661 0 R 3672 0 R 3682 0 R 3694 0 R 3707 0 R 3710 0 R 3712 0 R 3716 0 R 3729 0 R 3735 0 R 3740 0 R 3746 0 R 3750 0 R 3753 0 R 3758 0 R 3760 0 R 3764 0 R 3766 0 R 3770 0 R 3773 0 R 3776 0 R 3784 0 R 3786 0 R 3792 0 R 3795 0 R 3802 0 R 3806 0 R 3809 0 R 3812 0 R 3815 0 R 3819 0 R 3822 0 R 3827 0 R 3832 0 R 3835 0 R 3843 0 R 3847 0 R 3851 0 R 3853 0 R 3856 0 R 3859 0 R 3864 0 R 3871 0 R 3877 0 R 3881 0 R 3884 0 R 3891 0 R 3899 0 R 3903 0 R 3906 0 R 3908 0 R 3912 0
R 3917 0 R 3922 0 R 3925 0 R 3934 0 R 3939 0 R 3943 0 R 3946 0 R 3954 0 R 3959 0 R 3967 0 R 3972 0 R 3974 0 R 3980 0 R 3982 0 R 3987 0 R 3991 0 R 3996 0 R 4000 0 R 4012 0 R 4028 0 R 4043 0 R 4048 0 R 4051 0 R 4054 0 R 4060 0 R 4065 0 R 4067 0 R 4069 0 R 4071 0 R 4073 0 R 4075 0 R 4084 0 R 4088 0 R 4092 0 R 4096 0 R 4098 0 R 4105 0 R 4115 0 R 4124 0 R 4127 0 R 4130 0 R 4132 0 R 4139 0 R 4146 0 R 4156 0 R 4160 0 R 4163 0 R 4167 0 R 4170 0 R 4176 0 R 4179 0 R 4194 0 R 4199 0 R 4222 0 R 4226 0 R 4233 0 R 4244 0 R 4253 0 R 4256 0 R 4259 0 R 4262 0 R 4278 0 R 4283 0 R 4290 0 R]
>>
endobj
4 0 obj
@@ -187,11 +187,11 @@ endobj
<< /Type /Font
/BaseFont /71be00+NotoSerif
/Subtype /TrueType
-/FontDescriptor 4885 0 R
+/FontDescriptor 5029 0 R
/FirstChar 32
/LastChar 255
-/Widths 4887 0 R
-/ToUnicode 4886 0 R
+/Widths 5031 0 R
+/ToUnicode 5030 0 R
>>
endobj
11 0 obj
@@ -1750,7 +1750,7 @@ endobj
/F1.0 10 0 R
>>
>>
-/Annots [4204 0 R 4205 0 R 4206 0 R 4207 0 R 4208 0 R 4209 0 R 4210 0 R 4211 0 R 4212 0 R 4213 0 R 4214 0 R 4215 0 R 4216 0 R 4217 0 R 4218 0 R 4219 0 R 4220 0 R 4221 0 R 4222 0 R 4223 0 R 4224 0 R 4225 0 R 4226 0 R 4227 0 R 4228 0 R 4229 0 R 4230 0 R 4231 0 R 4232 0 R 4233 0 R 4234 0 R 4235 0 R 4236 0 R 4237 0 R 4238 0 R 4239 0 R 4240 0 R 4241 0 R 4242 0 R 4243 0 R 4244 0 R 4245 0 R 4246 0 R 4247 0 R 4248 0 R 4249 0 R 4250 0 R 4251 0 R 4252 0 R 4253 0 R 4254 0 R 4255 0 R 4256 0 R 4257 0 R 4258 0 R 4259 0 R 4260 0 R 4261 0 R 4262 0 R 4263 0 R 4264 0 R 4265 0 R 4266 0 R 4267 0 R 4268 0 R 4269 0 R 4270 0 R 4271 0 R 4272 0 R 4273 0 R 4274 0 R 4275 0 R 4276 0 R 4277 0 R 4278 0 R 4279 0 R]
+/Annots [4292 0 R 4293 0 R 4294 0 R 4295 0 R 4296 0 R 4297 0 R 4298 0 R 4299 0 R 4300 0 R 4301 0 R 4302 0 R 4303 0 R 4304 0 R 4305 0 R 4306 0 R 4307 0 R 4308 0 R 4309 0 R 4310 0 R 4311 0 R 4312 0 R 4313 0 R 4314 0 R 4315 0 R 4316 0 R 4317 0 R 4318 0 R 4319 0 R 4320 0 R 4321 0 R 4322 0 R 4323 0 R 4324 0 R 4325 0 R 4326 0 R 4327 0 R 4328 0 R 4329 0 R 4330 0 R 4331 0 R 4332 0 R 4333 0 R 4334 0 R 4335 0 R 4336 0 R 4337 0 R 4338 0 R 4339 0 R 4340 0 R 4341 0 R 4342 0 R 4343 0 R 4344 0 R 4345 0 R 4346 0 R 4347 0 R 4348 0 R 4349 0 R 4350 0 R 4351 0 R 4352 0 R 4353 0 R 4354 0 R 4355 0 R 4356 0 R 4357 0 R 4358 0 R 4359 0 R 4360 0 R 4361 0 R 4362 0 R 4363 0 R 4364 0 R 4365 0 R 4366 0 R 4367 0 R]
>>
endobj
13 0 obj
@@ -3417,7 +3417,7 @@ endobj
/Font << /F1.0 10 0 R
>>
>>
-/Annots [4280 0 R 4281 0 R 4282 0 R 4283 0 R 4284 0 R 4285 0 R 4286 0 R 4287 0 R 4288 0 R 4289 0 R 4290 0 R 4291 0 R 4292 0 R 4293 0 R 4294 0 R 4295 0 R 4296 0 R 4297 0 R 4298 0 R 4299 0 R 4300 0 R 4301 0 R 4302 0 R 4303 0 R 4304 0 R 4305 0 R 4306 0 R 4307 0 R 4308 0 R 4309 0 R 4310 0 R 4311 0 R 4312 0 R 4313 0 R 4314 0 R 4315 0 R 4316 0 R 4317 0 R 4318 0 R 4319 0 R 4320 0 R 4321 0 R 4322 0 R 4323 0 R 4324 0 R 4325 0 R 4326 0 R 4327 0 R 4328 0 R 4329 0 R 4330 0 R 4331 0 R 4332 0 R 4333 0 R 4334 0 R 4335 0 R 4336 0 R 4337 0 R 4338 0 R 4339 0 R 4340 0 R 4341 0 R 4342 0 R 4343 0 R 4344 0 R 4345 0 R 4346 0 R 4347 0 R 4348 0 R 4349 0 R 4350 0 R 4351 0 R 4352 0 R 4353 0 R 4354 0 R 4355 0 R 4356 0 R 4357 0 R 4358 0 R 4359 0 R 4360 0 R 4361 0 R]
+/Annots [4368 0 R 4369 0 R 4370 0 R 4371 0 R 4372 0 R 4373 0 R 4374 0 R 4375 0 R 4376 0 R 4377 0 R 4378 0 R 4379 0 R 4380 0 R 4381 0 R 4382 0 R 4383 0 R 4384 0 R 4385 0 R 4386 0 R 4387 0 R 4388 0 R 4389 0 R 4390 0 R 4391 0 R 4392 0 R 4393 0 R 4394 0 R 4395 0 R 4396 0 R 4397 0 R 4398 0 R 4399 0 R 4400 0 R 4401 0 R 4402 0 R 4403 0 R 4404 0 R 4405 0 R 4406 0 R 4407 0 R 4408 0 R 4409 0 R 4410 0 R 4411 0 R 4412 0 R 4413 0 R 4414 0 R 4415 0 R 4416 0 R 4417 0 R 4418 0 R 4419 0 R 4420 0 R 4421 0 R 4422 0 R 4423 0 R 4424 0 R 4425 0 R 4426 0 R 4427 0 R 4428 0 R 4429 0 R 4430 0 R 4431 0 R 4432 0 R 4433 0 R 4434 0 R 4435 0 R 4436 0 R 4437 0 R 4438 0 R 4439 0 R 4440 0 R 4441 0 R 4442 0 R 4443 0 R 4444 0 R 4445 0 R 4446 0 R 4447 0 R 4448 0 R 4449 0 R]
>>
endobj
15 0 obj
@@ -5084,7 +5084,7 @@ endobj
/Font << /F1.0 10 0 R
>>
>>
-/Annots [4362 0 R 4363 0 R 4364 0 R 4365 0 R 4366 0 R 4367 0 R 4368 0 R 4369 0 R 4370 0 R 4371 0 R 4372 0 R 4373 0 R 4374 0 R 4375 0 R 4376 0 R 4377 0 R 4378 0 R 4379 0 R 4380 0 R 4381 0 R 4382 0 R 4383 0 R 4384 0 R 4385 0 R 4386 0 R 4387 0 R 4388 0 R 4389 0 R 4390 0 R 4391 0 R 4392 0 R 4393 0 R 4394 0 R 4395 0 R 4396 0 R 4397 0 R 4398 0 R 4399 0 R 4400 0 R 4401 0 R 4402 0 R 4403 0 R 4404 0 R 4405 0 R 4406 0 R 4407 0 R 4408 0 R 4409 0 R 4410 0 R 4411 0 R 4412 0 R 4413 0 R 4414 0 R 4415 0 R 4416 0 R 4417 0 R 4418 0 R 4419 0 R 4420 0 R 4421 0 R 4422 0 R 4423 0 R 4424 0 R 4425 0 R 4426 0 R 4427 0 R 4428 0 R 4429 0 R 4430 0 R 4431 0 R 4432 0 R 4433 0 R 4434 0 R 4435 0 R 4436 0 R 4437 0 R 4438 0 R 4439 0 R 4440 0 R 4441 0 R 4442 0 R 4443 0 R]
+/Annots [4450 0 R 4451 0 R 4452 0 R 4453 0 R 4454 0 R 4455 0 R 4456 0 R 4457 0 R 4458 0 R 4459 0 R 4460 0 R 4461 0 R 4462 0 R 4463 0 R 4464 0 R 4465 0 R 4466 0 R 4467 0 R 4468 0 R 4469 0 R 4470 0 R 4471 0 R 4472 0 R 4473 0 R 4474 0 R 4475 0 R 4476 0 R 4477 0 R 4478 0 R 4479 0 R 4480 0 R 4481 0 R 4482 0 R 4483 0 R 4484 0 R 4485 0 R 4486 0 R 4487 0 R 4488 0 R 4489 0 R 4490 0 R 4491 0 R 4492 0 R 4493 0 R 4494 0 R 4495 0 R 4496 0 R 4497 0 R 4498 0 R 4499 0 R 4500 0 R 4501 0 R 4502 0 R 4503 0 R 4504 0 R 4505 0 R 4506 0 R 4507 0 R 4508 0 R 4509 0 R 4510 0 R 4511 0 R 4512 0 R 4513 0 R 4514 0 R 4515 0 R 4516 0 R 4517 0 R 4518 0 R 4519 0 R 4520 0 R 4521 0 R 4522 0 R 4523 0 R 4524 0 R 4525 0 R 4526 0 R 4527 0 R 4528 0 R 4529 0 R 4530 0 R 4531 0 R]
>>
endobj
17 0 obj
@@ -6751,11 +6751,11 @@ endobj
/Font << /F1.0 10 0 R
>>
>>
-/Annots [4444 0 R 4445 0 R 4446 0 R 4447 0 R 4448 0 R 4449 0 R 4450 0 R 4451 0 R 4452 0 R 4453 0 R 4454 0 R 4455 0 R 4456 0 R 4457 0 R 4458 0 R 4459 0 R 4460 0 R 4461 0 R 4462 0 R 4463 0 R 4464 0 R 4465 0 R 4466 0 R 4467 0 R 4468 0 R 4469 0 R 4470 0 R 4471 0 R 4472 0 R 4473 0 R 4474 0 R 4475 0 R 4476 0 R 4477 0 R 4478 0 R 4479 0 R 4480 0 R 4481 0 R 4482 0 R 4483 0 R 4484 0 R 4485 0 R 4486 0 R 4487 0 R 4488 0 R 4489 0 R 4490 0 R 4491 0 R 4492 0 R 4493 0 R 4494 0 R 4495 0 R 4496 0 R 4497 0 R 4498 0 R 4499 0 R 4500 0 R 4501 0 R 4502 0 R 4503 0 R 4504 0 R 4505 0 R 4506 0 R 4507 0 R 4508 0 R 4509 0 R 4510 0 R 4511 0 R 4512 0 R 4513 0 R 4514 0 R 4515 0 R 4516 0 R 4517 0 R 4518 0 R 4519 0 R 4520 0 R 4521 0 R 4522 0 R 4523 0 R 4524 0 R 4525 0 R]
+/Annots [4532 0 R 4533 0 R 4534 0 R 4535 0 R 4536 0 R 4537 0 R 4538 0 R 4539 0 R 4540 0 R 4541 0 R 4542 0 R 4543 0 R 4544 0 R 4545 0 R 4546 0 R 4547 0 R 4548 0 R 4549 0 R 4550 0 R 4551 0 R 4552 0 R 4553 0 R 4554 0 R 4555 0 R 4556 0 R 4557 0 R 4558 0 R 4559 0 R 4560 0 R 4561 0 R 4562 0 R 4563 0 R 4564 0 R 4565 0 R 4566 0 R 4567 0 R 4568 0 R 4569 0 R 4570 0 R 4571 0 R 4572 0 R 4573 0 R 4574 0 R 4575 0 R 4576 0 R 4577 0 R 4578 0 R 4579 0 R 4580 0 R 4581 0 R 4582 0 R 4583 0 R 4584 0 R 4585 0 R 4586 0 R 4587 0 R 4588 0 R 4589 0 R 4590 0 R 4591 0 R 4592 0 R 4593 0 R 4594 0 R 4595 0 R 4596 0 R 4597 0 R 4598 0 R 4599 0 R 4600 0 R 4601 0 R 4602 0 R 4603 0 R 4604 0 R 4605 0 R 4606 0 R 4607 0 R 4608 0 R 4609 0 R 4610 0 R 4611 0 R 4612 0 R 4613 0 R]
>>
endobj
19 0 obj
-<< /Length 29395
+<< /Length 29486
>>
stream
q
@@ -8338,7 +8338,18 @@ ET
BT
48.24 73.956 Td
/F1.0 10.5 Tf
-[<5a6f6f4b> 20.0195 <6565706572>] TJ
+[<50726f636564757265204672> 20.0195 <616d65776f726b2028507632293a20>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+195.2713 73.956 Td
+/F1.0 10.5 Tf
+[<4842> 20.0195 <4153452d3132343339>] TJ
ET
0.0 0.0 0.0 SCN
@@ -8347,9 +8358,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-106.5361 73.956 Td
+266.8711 73.956 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8378,7 +8389,7 @@ ET
BT
60.24 55.476 Td
/F1.0 10.5 Tf
-[<3137362e205573696e67206578697374696e67205a6f6f4b> 20.0195 <656570657220656e73656d626c65>] TJ
+<3137362e2050726f63656475726573> Tj
ET
0.0 0.0 0.0 SCN
@@ -8387,9 +8398,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-261.5266 55.476 Td
+143.9476 55.476 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8407,7 +8418,7 @@ ET
BT
529.4315 55.476 Td
/F1.0 10.5 Tf
-<363530> Tj
+<363439> Tj
ET
0.0 0.0 0.0 SCN
@@ -8429,11 +8440,11 @@ endobj
/Font << /F1.0 10 0 R
>>
>>
-/Annots [4526 0 R 4527 0 R 4528 0 R 4529 0 R 4530 0 R 4531 0 R 4532 0 R 4533 0 R 4534 0 R 4535 0 R 4536 0 R 4537 0 R 4538 0 R 4539 0 R 4540 0 R 4541 0 R 4542 0 R 4543 0 R 4544 0 R 4545 0 R 4546 0 R 4547 0 R 4548 0 R 4549 0 R 4550 0 R 4551 0 R 4552 0 R 4553 0 R 4554 0 R 4555 0 R 4556 0 R 4557 0 R 4558 0 R 4559 0 R 4560 0 R 4561 0 R 4562 0 R 4563 0 R 4564 0 R 4565 0 R 4566 0 R 4567 0 R 4568 0 R 4569 0 R 4570 0 R 4571 0 R 4572 0 R 4573 0 R 4574 0 R 4575 0 R 4576 0 R 4577 0 R 4578 0 R 4579 0 R 4580 0 R 4581 0 R 4582 0 R 4583 0 R 4584 0 R 4585 0 R 4586 0 R 4587 0 R 4588 0 R 4589 0 R 4590 0 R 4591 0 R 4592 0 R 4593 0 R 4594 0 R 4595 0 R 4596 0 R 4597 0 R 4598 0 R 4599 0 R 4600 0 R 4601 0 R 4602 0 R 4603 0 R 4604 0 R 4605 0 R 4606 0 R 4607 0 R 4608 0 R 4609 0 R]
+/Annots [4614 0 R 4615 0 R 4616 0 R 4617 0 R 4618 0 R 4619 0 R 4620 0 R 4621 0 R 4622 0 R 4623 0 R 4624 0 R 4625 0 R 4626 0 R 4627 0 R 4628 0 R 4629 0 R 4630 0 R 4631 0 R 4632 0 R 4633 0 R 4634 0 R 4635 0 R 4636 0 R 4637 0 R 4638 0 R 4639 0 R 4640 0 R 4641 0 R 4642 0 R 4643 0 R 4644 0 R 4645 0 R 4646 0 R 4647 0 R 4648 0 R 4649 0 R 4650 0 R 4651 0 R 4652 0 R 4653 0 R 4654 0 R 4655 0 R 4656 0 R 4657 0 R 4658 0 R 4659 0 R 4660 0 R 4661 0 R 4662 0 R 4663 0 R 4664 0 R 4665 0 R 4666 0 R 4667 0 R 4668 0 R 4669 0 R 4670 0 R 4671 0 R 4672 0 R 4673 0 R 4674 0 R 4675 0 R 4676 0 R 4677 0 R 4678 0 R 4679 0 R 4680 0 R 4681 0 R 4682 0 R 4683 0 R 4684 0 R 4685 0 R 4686 0 R 4687 0 R 4688 0 R 4689 0 R 4690 0 R 4691 0 R 4692 0 R 4693 0 R 4694 0 R 4695 0 R 4696 0 R 4697 0 R 4698 0 R 4699 0 R]
>>
endobj
21 0 obj
-<< /Length 15523
+<< /Length 28610
>>
stream
q
@@ -8445,7 +8456,7 @@ q
BT
60.24 794.676 Td
/F1.0 10.5 Tf
-[<3137372e2053> 20.0195 <41534c2041> 20.0195 <757468656e7469636174696f6e2077697468205a6f6f4b> 20.0195 <6565706572>] TJ
+<3137372e2053756270726f63656475726573> Tj
ET
0.0 0.0 0.0 SCN
@@ -8454,9 +8465,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-272.2156 794.676 Td
+159.9811 794.676 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8474,7 +8485,7 @@ ET
BT
529.4315 794.676 Td
/F1.0 10.5 Tf
-<363531> Tj
+<363532> Tj
ET
0.0 0.0 0.0 SCN
@@ -8483,9 +8494,9 @@ ET
0.2 0.2 0.2 SCN
BT
-48.24 776.196 Td
+60.24 776.196 Td
/F1.0 10.5 Tf
-<436f6d6d756e697479> Tj
+<3137382e2050726f6365647572654578656375746f72> Tj
ET
0.0 0.0 0.0 SCN
@@ -8494,9 +8505,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-111.8806 776.196 Td
+181.3591 776.196 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8514,7 +8525,7 @@ ET
BT
529.4315 776.196 Td
/F1.0 10.5 Tf
-<363538> Tj
+<363533> Tj
ET
0.0 0.0 0.0 SCN
@@ -8525,7 +8536,7 @@ ET
BT
60.24 757.716 Td
/F1.0 10.5 Tf
-<3137382e204465636973696f6e73> Tj
+<3137392e204e6f6e636573> Tj
ET
0.0 0.0 0.0 SCN
@@ -8534,9 +8545,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-133.2586 757.716 Td
+122.5696 757.716 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8554,7 +8565,7 @@ ET
BT
529.4315 757.716 Td
/F1.0 10.5 Tf
-<363539> Tj
+<363534> Tj
ET
0.0 0.0 0.0 SCN
@@ -8565,7 +8576,7 @@ ET
BT
60.24 739.236 Td
/F1.0 10.5 Tf
-<3137392e20436f6d6d756e69747920526f6c6573> Tj
+[<3138302e2057> 49.8047 <6169742f57> 49.8047 <616b> 20.0195 <652f53757370656e642f5969656c64>] TJ
ET
0.0 0.0 0.0 SCN
@@ -8574,9 +8585,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-176.0146 739.236 Td
+213.4261 739.236 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8594,7 +8605,7 @@ ET
BT
529.4315 739.236 Td
/F1.0 10.5 Tf
-<363630> Tj
+<363535> Tj
ET
0.0 0.0 0.0 SCN
@@ -8605,7 +8616,7 @@ ET
BT
60.24 720.756 Td
/F1.0 10.5 Tf
-<3138302e20436f6d6d6974204d65737361676520666f726d6174> Tj
+<3138312e204c6f636b696e67> Tj
ET
0.0 0.0 0.0 SCN
@@ -8614,9 +8625,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-208.0816 720.756 Td
+127.9141 720.756 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8634,7 +8645,7 @@ ET
BT
529.4315 720.756 Td
/F1.0 10.5 Tf
-<363631> Tj
+<363536> Tj
ET
0.0 0.0 0.0 SCN
@@ -8643,9 +8654,9 @@ ET
0.2 0.2 0.2 SCN
BT
-48.24 702.276 Td
+60.24 702.276 Td
/F1.0 10.5 Tf
-<417070656e646978> Tj
+<3138322e2050726f636564757265205479706573> Tj
ET
0.0 0.0 0.0 SCN
@@ -8654,9 +8665,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-101.1916 702.276 Td
+170.6701 702.276 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8674,7 +8685,7 @@ ET
BT
529.4315 702.276 Td
/F1.0 10.5 Tf
-<363632> Tj
+<363537> Tj
ET
0.0 0.0 0.0 SCN
@@ -8685,7 +8696,7 @@ ET
BT
60.24 683.796 Td
/F1.0 10.5 Tf
-<417070656e64697820413a20436f6e747269627574696e6720746f20446f63756d656e746174696f6e> Tj
+<3138332e205265666572656e636573> Tj
ET
0.0 0.0 0.0 SCN
@@ -8694,9 +8705,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-282.9046 683.796 Td
+138.6031 683.796 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8714,7 +8725,7 @@ ET
BT
529.4315 683.796 Td
/F1.0 10.5 Tf
-<363633> Tj
+<363538> Tj
ET
0.0 0.0 0.0 SCN
@@ -8723,9 +8734,9 @@ ET
0.2 0.2 0.2 SCN
BT
-60.24 665.316 Td
+48.24 665.316 Td
/F1.0 10.5 Tf
-[<417070656e64697820423a2046> 69.8242 <41> 20.0195 <51>] TJ
+<414d7632204465736372697074696f6e20666f722044657673> Tj
ET
0.0 0.0 0.0 SCN
@@ -8734,9 +8745,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-149.2921 665.316 Td
+186.7036 665.316 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8754,7 +8765,7 @@ ET
BT
529.4315 665.316 Td
/F1.0 10.5 Tf
-<363734> Tj
+<363539> Tj
ET
0.0 0.0 0.0 SCN
@@ -8765,7 +8776,7 @@ ET
BT
60.24 646.836 Td
/F1.0 10.5 Tf
-<417070656e64697820433a206862636b20496e204465707468> Tj
+[<3138342e204261636b> 20.0195 <67726f756e64>] TJ
ET
0.0 0.0 0.0 SCN
@@ -8774,9 +8785,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-197.3926 646.836 Td
+143.9476 646.836 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8794,7 +8805,7 @@ ET
BT
529.4315 646.836 Td
/F1.0 10.5 Tf
-<363737> Tj
+<363630> Tj
ET
0.0 0.0 0.0 SCN
@@ -8805,7 +8816,7 @@ ET
BT
60.24 628.356 Td
/F1.0 10.5 Tf
-[<417070656e64697820443a2041> 20.0195 <636365737320436f6e74726f6c204d6174726978>] TJ
+[<3138352e204e65772053> 20.0195 <797374656d>] TJ
ET
0.0 0.0 0.0 SCN
@@ -8814,9 +8825,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-234.8041 628.356 Td
+149.2921 628.356 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8834,7 +8845,7 @@ ET
BT
529.4315 628.356 Td
/F1.0 10.5 Tf
-<363831> Tj
+<363631> Tj
ET
0.0 0.0 0.0 SCN
@@ -8845,7 +8856,7 @@ ET
BT
60.24 609.876 Td
/F1.0 10.5 Tf
-<417070656e64697820453a20436f6d7072657373696f6e20616e64204461746120426c6f636b20456e636f64696e6720496e204842617365> Tj
+<3138362e2050726f636564757265732044657461696c> Tj
ET
0.0 0.0 0.0 SCN
@@ -8854,9 +8865,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-368.4166 609.876 Td
+176.0146 609.876 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8874,7 +8885,7 @@ ET
BT
529.4315 609.876 Td
/F1.0 10.5 Tf
-<363837> Tj
+<363632> Tj
ET
0.0 0.0 0.0 SCN
@@ -8885,7 +8896,7 @@ ET
BT
60.24 591.396 Td
/F1.0 10.5 Tf
-<417070656e64697820463a2053514c206f766572204842617365> Tj
+<3138372e205549> Tj
ET
0.0 0.0 0.0 SCN
@@ -8894,9 +8905,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-208.0816 591.396 Td
+95.8471 591.396 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8914,7 +8925,7 @@ ET
BT
529.4315 591.396 Td
/F1.0 10.5 Tf
-<363938> Tj
+<363634> Tj
ET
0.0 0.0 0.0 SCN
@@ -8925,7 +8936,7 @@ ET
BT
60.24 572.916 Td
/F1.0 10.5 Tf
-[<417070656e64697820473a2059> 29.7852 <435342>] TJ
+<3138382e204c6f6767696e67> Tj
ET
0.0 0.0 0.0 SCN
@@ -8934,9 +8945,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-154.6366 572.916 Td
+127.9141 572.916 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8954,7 +8965,7 @@ ET
BT
529.4315 572.916 Td
/F1.0 10.5 Tf
-<363939> Tj
+<363635> Tj
ET
0.0 0.0 0.0 SCN
@@ -8965,7 +8976,7 @@ ET
BT
60.24 554.436 Td
/F1.0 10.5 Tf
-<417070656e64697820483a204846696c6520666f726d6174> Tj
+<3138392e20496d706c656d656e746174696f6e204e6f746573> Tj
ET
0.0 0.0 0.0 SCN
@@ -8974,9 +8985,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-192.0481 554.436 Td
+197.3926 554.436 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -8994,7 +9005,7 @@ ET
BT
529.4315 554.436 Td
/F1.0 10.5 Tf
-<373030> Tj
+<363636> Tj
ET
0.0 0.0 0.0 SCN
@@ -9005,7 +9016,7 @@ ET
BT
60.24 535.956 Td
/F1.0 10.5 Tf
-<417070656e64697820493a204f7468657220496e666f726d6174696f6e2041626f7574204842617365> Tj
+<3139302e204e657720436f6e66696773> Tj
ET
0.0 0.0 0.0 SCN
@@ -9014,9 +9025,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-282.9046 535.956 Td
+149.2921 535.956 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9034,7 +9045,7 @@ ET
BT
529.4315 535.956 Td
/F1.0 10.5 Tf
-<373039> Tj
+<363637> Tj
ET
0.0 0.0 0.0 SCN
@@ -9045,7 +9056,7 @@ ET
BT
60.24 517.476 Td
/F1.0 10.5 Tf
-<417070656e646978204a3a20484261736520486973746f7279> Tj
+[<3139312e2054> 29.7852 <6f6f6c73>] TJ
ET
0.0 0.0 0.0 SCN
@@ -9054,9 +9065,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-197.3926 517.476 Td
+111.8806 517.476 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9074,7 +9085,7 @@ ET
BT
529.4315 517.476 Td
/F1.0 10.5 Tf
-<373130> Tj
+<363638> Tj
ET
0.0 0.0 0.0 SCN
@@ -9083,9 +9094,9 @@ ET
0.2 0.2 0.2 SCN
BT
-60.24 498.996 Td
+48.24 498.996 Td
/F1.0 10.5 Tf
-[<417070656e646978204b3a20484261736520616e64207468652041706163686520536f6674776172652046> 40.0391 <6f756e646174696f6e>] TJ
+[<5a6f6f4b> 20.0195 <6565706572>] TJ
ET
0.0 0.0 0.0 SCN
@@ -9094,9 +9105,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-347.0386 498.996 Td
+106.5361 498.996 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9114,7 +9125,7 @@ ET
BT
529.4315 498.996 Td
/F1.0 10.5 Tf
-<373131> Tj
+<363639> Tj
ET
0.0 0.0 0.0 SCN
@@ -9125,7 +9136,7 @@ ET
BT
60.24 480.516 Td
/F1.0 10.5 Tf
-<417070656e646978204c3a20417061636865204842617365204f726361> Tj
+[<3139322e205573696e67206578697374696e67205a6f6f4b> 20.0195 <656570657220656e73656d626c65>] TJ
ET
0.0 0.0 0.0 SCN
@@ -9134,9 +9145,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-224.1151 480.516 Td
+261.5266 480.516 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9154,7 +9165,7 @@ ET
BT
529.4315 480.516 Td
/F1.0 10.5 Tf
-<373132> Tj
+<363731> Tj
ET
0.0 0.0 0.0 SCN
@@ -9165,7 +9176,7 @@ ET
BT
60.24 462.036 Td
/F1.0 10.5 Tf
-[<417070656e646978204d3a20456e61626c696e67204461707065722d6c696b> 20.0195 <65205472> 20.0195 <6163696e6720696e204842617365>] TJ
+[<3139332e2053> 20.0195 <41534c2041> 20.0195 <757468656e7469636174696f6e2077697468205a6f6f4b> 20.0195 <6565706572>] TJ
ET
0.0 0.0 0.0 SCN
@@ -9174,9 +9185,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-325.6606 462.036 Td
+272.2156 462.036 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9194,7 +9205,7 @@ ET
BT
529.4315 462.036 Td
/F1.0 10.5 Tf
-<373133> Tj
+<363732> Tj
ET
0.0 0.0 0.0 SCN
@@ -9203,9 +9214,9 @@ ET
0.2 0.2 0.2 SCN
BT
-60.24 443.556 Td
+48.24 443.556 Td
/F1.0 10.5 Tf
-<3138312e20436c69656e74204d6f64696669636174696f6e73> Tj
+<436f6d6d756e697479> Tj
ET
0.0 0.0 0.0 SCN
@@ -9214,9 +9225,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-186.7036 443.556 Td
+111.8806 443.556 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9234,7 +9245,7 @@ ET
BT
529.4315 443.556 Td
/F1.0 10.5 Tf
-<373135> Tj
+<363739> Tj
ET
0.0 0.0 0.0 SCN
@@ -9245,7 +9256,7 @@ ET
BT
60.24 425.076 Td
/F1.0 10.5 Tf
-[<3138322e205472> 20.0195 <6163696e672066726f6d204842617365205368656c6c>] TJ
+<3139342e204465636973696f6e73> Tj
ET
0.0 0.0 0.0 SCN
@@ -9254,9 +9265,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-213.4261 425.076 Td
+133.2586 425.076 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9274,7 +9285,7 @@ ET
BT
529.4315 425.076 Td
/F1.0 10.5 Tf
-<373136> Tj
+<363830> Tj
ET
0.0 0.0 0.0 SCN
@@ -9285,7 +9296,7 @@ ET
BT
60.24 406.596 Td
/F1.0 10.5 Tf
-<417070656e646978204e3a20302e3935205250432053706563696669636174696f6e> Tj
+<3139352e20436f6d6d756e69747920526f6c6573> Tj
ET
0.0 0.0 0.0 SCN
@@ -9294,9 +9305,9 @@ ET
0.6627 0.6627 0.6627 SCN
BT
-240.1486 406.596 Td
+176.0146 406.596 Td
/F1.0 10.5 Tf
-<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
ET
0.0 0.0 0.0 SCN
@@ -9314,7 +9325,727 @@ ET
BT
529.4315 406.596 Td
/F1.0 10.5 Tf
-<373137> Tj
+<363831> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 388.116 Td
+/F1.0 10.5 Tf
+<3139362e20436f6d6d6974204d65737361676520666f726d6174> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+208.0816 388.116 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 388.116 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 388.116 Td
+/F1.0 10.5 Tf
+<363832> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+48.24 369.636 Td
+/F1.0 10.5 Tf
+<417070656e646978> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+101.1916 369.636 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 369.636 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 369.636 Td
+/F1.0 10.5 Tf
+<363833> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 351.156 Td
+/F1.0 10.5 Tf
+<417070656e64697820413a20436f6e747269627574696e6720746f20446f63756d656e746174696f6e> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+282.9046 351.156 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 351.156 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 351.156 Td
+/F1.0 10.5 Tf
+<363834> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 332.676 Td
+/F1.0 10.5 Tf
+[<417070656e64697820423a2046> 69.8242 <41> 20.0195 <51>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+149.2921 332.676 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 332.676 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 332.676 Td
+/F1.0 10.5 Tf
+<363935> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 314.196 Td
+/F1.0 10.5 Tf
+<417070656e64697820433a206862636b20496e204465707468> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+197.3926 314.196 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 314.196 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 314.196 Td
+/F1.0 10.5 Tf
+<363938> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 295.716 Td
+/F1.0 10.5 Tf
+[<417070656e64697820443a2041> 20.0195 <636365737320436f6e74726f6c204d6174726978>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+234.8041 295.716 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 295.716 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 295.716 Td
+/F1.0 10.5 Tf
+<373032> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 277.236 Td
+/F1.0 10.5 Tf
+<417070656e64697820453a20436f6d7072657373696f6e20616e64204461746120426c6f636b20456e636f64696e6720496e204842617365> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+368.4166 277.236 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 277.236 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 277.236 Td
+/F1.0 10.5 Tf
+<373038> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 258.756 Td
+/F1.0 10.5 Tf
+<417070656e64697820463a2053514c206f766572204842617365> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+208.0816 258.756 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 258.756 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 258.756 Td
+/F1.0 10.5 Tf
+<373139> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 240.276 Td
+/F1.0 10.5 Tf
+[<417070656e64697820473a2059> 29.7852 <435342>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+154.6366 240.276 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 240.276 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 240.276 Td
+/F1.0 10.5 Tf
+<373230> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 221.796 Td
+/F1.0 10.5 Tf
+<417070656e64697820483a204846696c6520666f726d6174> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+192.0481 221.796 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 221.796 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 221.796 Td
+/F1.0 10.5 Tf
+<373231> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 203.316 Td
+/F1.0 10.5 Tf
+<417070656e64697820493a204f7468657220496e666f726d6174696f6e2041626f7574204842617365> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+282.9046 203.316 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 203.316 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 203.316 Td
+/F1.0 10.5 Tf
+<373330> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 184.836 Td
+/F1.0 10.5 Tf
+<417070656e646978204a3a20484261736520486973746f7279> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+197.3926 184.836 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 184.836 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 184.836 Td
+/F1.0 10.5 Tf
+<373331> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 166.356 Td
+/F1.0 10.5 Tf
+[<417070656e646978204b3a20484261736520616e64207468652041706163686520536f6674776172652046> 40.0391 <6f756e646174696f6e>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+347.0386 166.356 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 166.356 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 166.356 Td
+/F1.0 10.5 Tf
+<373332> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 147.876 Td
+/F1.0 10.5 Tf
+<417070656e646978204c3a20417061636865204842617365204f726361> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+224.1151 147.876 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 147.876 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 147.876 Td
+/F1.0 10.5 Tf
+<373333> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 129.396 Td
+/F1.0 10.5 Tf
+[<417070656e646978204d3a20456e61626c696e67204461707065722d6c696b> 20.0195 <65205472> 20.0195 <6163696e6720696e204842617365>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+325.6606 129.396 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 129.396 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 129.396 Td
+/F1.0 10.5 Tf
+<373334> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 110.916 Td
+/F1.0 10.5 Tf
+<3139372e20436c69656e74204d6f64696669636174696f6e73> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+186.7036 110.916 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 110.916 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 110.916 Td
+/F1.0 10.5 Tf
+<373336> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 92.436 Td
+/F1.0 10.5 Tf
+[<3139382e205472> 20.0195 <6163696e672066726f6d204842617365205368656c6c>] TJ
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+213.4261 92.436 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 92.436 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 92.436 Td
+/F1.0 10.5 Tf
+<373337> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+60.24 73.956 Td
+/F1.0 10.5 Tf
+<417070656e646978204e3a20302e3935205250432053706563696669636174696f6e> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+0.6627 0.6627 0.6627 scn
+0.6627 0.6627 0.6627 SCN
+
+BT
+240.1486 73.956 Td
+/F1.0 10.5 Tf
+<2e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e202e20> Tj
+ET
+
+0.0 0.0 0.0 SCN
+0.0 0.0 0.0 scn
+
+BT
+528.7516 73.956 Td
+/F1.0 2.625 Tf
+<ca> Tj
+ET
+
+0.2 0.2 0.2 scn
+0.2 0.2 0.2 SCN
+
+BT
+529.4315 73.956 Td
+/F1.0 10.5 Tf
+<373338> Tj
ET
0.0 0.0 0.0 SCN
@@ -9336,7 +10067,7 @@ endobj
/Font << /F1.0 10 0 R
>>
>>
-/Annots [4610 0 R 4611 0 R 4612 0 R 4613 0 R 4614 0 R 4615 0 R 4616 0 R 4617 0 R 4618 0 R 4619 0 R 4620 0 R 4621 0 R 4622 0 R 4623 0 R 4624 0 R 4625 0 R 4626 0 R 4627 0 R 4628 0 R 4629 0 R 4630 0 R 4631 0 R 4632 0 R 4633 0 R 4634 0 R 4635 0 R 4636 0 R 4637 0 R 4638 0 R 4639 0 R 4640 0 R 4641 0 R 4642 0 R 4643 0 R 4644 0 R 4645 0 R 4646 0 R 4647 0 R 4648 0 R 4649 0 R 4650 0 R 4651 0 R 4652 0 R 4653 0 R]
+/Annots [4700 0 R 4701 0 R 4702 0 R 4703 0 R 4704 0 R 4705 0 R 4706 0 R 4707 0 R 4708 0 R 4709 0 R 4710 0 R 4711 0 R 4712 0 R 4713 0 R 4714 0 R 4715 0 R 4716 0 R 4717 0 R 4718 0 R 4719 0 R 4720 0 R 4721 0 R 4722 0 R 4723 0 R 4724 0 R 4725 0 R 4726 0 R 4727 0 R 4728 0 R 4729 0 R 4730 0 R 4731 0 R 4732 0 R 4733 0 R 4734 0 R 4735 0 R 4736 0 R 4737 0 R 4738 0 R 4739 0 R 4740 0 R 4741 0 R 4742 0 R 4743 0 R 4744 0 R 4745 0 R 4746 0 R 4747 0 R 4748 0 R 4749 0 R 4750 0 R 4751 0 R 4752 0 R 4753 0 R 4754 0 R 4755 0 R 4756 0 R 4757 0 R 4758 0 R 4759 0 R 4760 0 R 4761 0 R 4762 0 R 4763 0 R 4764 0 R 4765 0 R 4766 0 R 4767 0 R 4768 0 R 4769 0 R 4770 0 R 4771 0 R 4772 0 R 4773 0 R 4774 0 R 4775 0 R 4776 0 R 4777 0 R 4778 0 R 4779 0 R]
>>
endobj
23 0 obj
@@ -10148,7 +10879,7 @@ endobj
/F4.0 35 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [30 0 R 31 0 R 32 0 R 34 0 R 36 0 R 37 0 R 39 0 R 40 0 R 41 0 R]
@@ -10163,7 +10894,7 @@ endobj
>>
endobj
27 0 obj
-<< /Kids [643 0 R 3670 0 R 1936 0 R 644 0 R 4032 0 R 1166 0 R 2551 0 R 3804 0 R]
+<< /Kids [643 0 R 3670 0 R 1936 0 R 644 0 R 4121 0 R 1166 0 R 2551 0 R 3894 0 R]
>>
endobj
28 0 obj
@@ -10171,13 +10902,13 @@ endobj
endobj
29 0 obj
<< /Type /Font
-/BaseFont /358635+NotoSerif-Bold
+/BaseFont /309344+NotoSerif-Bold
/Subtype /TrueType
-/FontDescriptor 4889 0 R
+/FontDescriptor 5033 0 R
/FirstChar 32
/LastChar 255
-/Widths 4891 0 R
-/ToUnicode 4890 0 R
+/Widths 5035 0 R
+/ToUnicode 5034 0 R
>>
endobj
30 0 obj
@@ -10215,13 +10946,13 @@ endobj
endobj
33 0 obj
<< /Type /Font
-/BaseFont /260f03+NotoSerif-Italic
+/BaseFont /fbe45d+NotoSerif-Italic
/Subtype /TrueType
-/FontDescriptor 4893 0 R
+/FontDescriptor 5037 0 R
/FirstChar 32
/LastChar 255
-/Widths 4895 0 R
-/ToUnicode 4894 0 R
+/Widths 5039 0 R
+/ToUnicode 5038 0 R
>>
endobj
34 0 obj
@@ -10239,11 +10970,11 @@ endobj
<< /Type /Font
/BaseFont /c7d210+mplus1mn-regular
/Subtype /TrueType
-/FontDescriptor 4897 0 R
+/FontDescriptor 5041 0 R
/FirstChar 32
/LastChar 255
-/Widths 4899 0 R
-/ToUnicode 4898 0 R
+/Widths 5043 0 R
+/ToUnicode 5042 0 R
>>
endobj
36 0 obj
@@ -10267,13 +10998,13 @@ endobj
endobj
38 0 obj
<< /Type /Font
-/BaseFont /34c70d+NotoSerif
+/BaseFont /6bc580+NotoSerif
/Subtype /TrueType
-/FontDescriptor 4901 0 R
+/FontDescriptor 5045 0 R
/FirstChar 32
/LastChar 255
-/Widths 4903 0 R
-/ToUnicode 4902 0 R
+/Widths 5047 0 R
+/ToUnicode 5046 0 R
>>
endobj
39 0 obj
@@ -10707,7 +11438,7 @@ endobj
/F5.1 45 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -10716,11 +11447,11 @@ endobj
<< /Type /Font
/BaseFont /26ec65+FontAwesome
/Subtype /TrueType
-/FontDescriptor 4905 0 R
+/FontDescriptor 5049 0 R
/FirstChar 32
/LastChar 255
-/Widths 4907 0 R
-/ToUnicode 4906 0 R
+/Widths 5051 0 R
+/ToUnicode 5050 0 R
>>
endobj
46 0 obj
@@ -10778,7 +11509,7 @@ endobj
/Font << /F2.0 29 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -10863,7 +11594,7 @@ endobj
/Font << /F2.0 29 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [52 0 R]
@@ -12436,7 +13167,7 @@ endobj
/F1.1 38 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [57 0 R 59 0 R]
@@ -13556,7 +14287,7 @@ endobj
/F1.0 10 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -14435,7 +15166,7 @@ endobj
/F3.0 33 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [64 0 R]
@@ -15225,7 +15956,7 @@ endobj
/Font << /F4.0 35 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -15944,7 +16675,7 @@ endobj
/Font << /F4.0 35 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -16820,7 +17551,7 @@ endobj
/F2.0 29 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [73 0 R 74 0 R 75 0 R 76 0 R]
@@ -17810,7 +18541,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [79 0 R]
@@ -18733,7 +19464,7 @@ endobj
/F4.0 35 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -20244,7 +20975,7 @@ endobj
/F4.0 35 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [84 0 R 86 0 R]
@@ -21592,7 +22323,7 @@ endobj
/F4.0 35 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [90 0 R]
@@ -22666,7 +23397,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -23389,7 +24120,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -24028,7 +24759,7 @@ endobj
/F4.0 35 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [97 0 R 98 0 R 99 0 R 101 0 R]
@@ -24272,7 +25003,7 @@ endobj
/Font << /F2.0 29 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [105 0 R 106 0 R 107 0 R 108 0 R]
@@ -25144,7 +25875,7 @@ endobj
/F4.0 35 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [112 0 R 113 0 R]
@@ -25424,7 +26155,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -26904,7 +27635,7 @@ endobj
/F5.1 45 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [120 0 R 121 0 R 122 0 R 123 0 R 124 0 R 125 0 R 126 0 R 128 0 R 129 0 R 130 0 R 131 0 R]
@@ -27944,7 +28675,7 @@ endobj
/F2.0 29 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [134 0 R 136 0 R]
@@ -28697,7 +29428,7 @@ endobj
/F2.0 29 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [139 0 R 141 0 R 142 0 R 143 0 R 144 0 R]
@@ -32199,7 +32930,7 @@ endobj
/F2.0 29 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [147 0 R 148 0 R 149 0 R 150 0 R 151 0 R 152 0 R 153 0 R]
@@ -33000,7 +33731,7 @@ endobj
/F2.0 29 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [159 0 R 160 0 R]
@@ -33011,7 +33742,7 @@ endobj
endobj
157 0 obj
<< /Limits [(__anchor-top) (adding.new.node)]
-/Names [(__anchor-top) 25 0 R (__indexterm-2002) 3564 0 R (__indexterm-2004) 3566 0 R (__indexterm-2006) 3568 0 R (__indexterm-2008) 3571 0 R (acid) 939 0 R (acl) 3369 0 R (add-metric-name-and-function-to-hadoop-compat-interface) 3664 0 R (add-the-implementation-to-both-hadoop-1-and-hadoop-2-compat-modules) 3666 0 R (add.metrics) 3662 0 R (adding-a-new-chapter-to-the-hbase-reference-guide) 3908 0 R (adding.new.node) 3112 0 R]
+/Names [(__anchor-top) 25 0 R (__indexterm-2002) 3564 0 R (__indexterm-2004) 3566 0 R (__indexterm-2006) 3568 0 R (__indexterm-2008) 3571 0 R (acid) 939 0 R (acl) 3369 0 R (add-metric-name-and-function-to-hadoop-compat-interface) 3664 0 R (add-the-implementation-to-both-hadoop-1-and-hadoop-2-compat-modules) 3666 0 R (add.metrics) 3662 0 R (adding-a-new-chapter-to-the-hbase-reference-guide) 3997 0 R (adding.new.node) 3112 0 R]
>>
endobj
158 0 obj
@@ -34155,7 +34886,7 @@ endobj
/F4.0 35 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [165 0 R 166 0 R 168 0 R]
@@ -34955,7 +35686,7 @@ endobj
/F5.1 45 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [173 0 R 174 0 R 175 0 R 177 0 R 178 0 R 180 0 R 181 0 R]
@@ -36427,7 +37158,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [184 0 R 185 0 R]
@@ -36852,7 +37583,7 @@ endobj
/F4.0 35 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -37508,7 +38239,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [191 0 R]
@@ -38275,7 +39006,7 @@ endobj
/F4.0 35 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [196 0 R]
@@ -38806,7 +39537,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -38822,7 +39553,7 @@ endobj
endobj
206 0 obj
<< /Limits [(getshortmidpointkey-an-optimization-for-data-index-block) (handling-of-errors-during-log-splitting)]
-/Names [(getshortmidpointkey-an-optimization-for-data-index-block) 4089 0 R (getting.involved) 3389 0 R (getting_started) 48 0 R (git.best.practices) 3667 0 R (git.patch.flow) 3722 0 R (goals) 4182 0 R (guide-for-hbase-committers) 3696 0 R (guidelines-for-deploying-a-coprocessor) 2389 0 R (guidelines-for-reporting-effective-issues) 3407 0 R (hadoop) 140 0 R (hadoop.native.lib) 4005 0 R (hadoop.policy.file) 383 0 R (handling-of-errors-during-log-splitting) 1681 0 R]
+/Names [(getshortmidpointkey-an-optimization-for-data-index-block) 4177 0 R (getting.involved) 3389 0 R (getting_started) 48 0 R (git.best.practices) 3667 0 R (git.patch.flow) 3722 0 R (goals) 4270 0 R (guide-for-hbase-committers) 3696 0 R (guidelines-for-deploying-a-coprocessor) 2389 0 R (guidelines-for-reporting-effective-issues) 3407 0 R (hadoop) 140 0 R (hadoop.native.lib) 4094 0 R (hadoop.policy.file) 383 0 R (handling-of-errors-during-log-splitting) 1681 0 R]
>>
endobj
207 0 obj
@@ -39352,7 +40083,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -39874,7 +40605,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -40509,7 +41240,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -41018,7 +41749,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -41549,7 +42280,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -42128,7 +42859,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [252 0 R 253 0 R]
@@ -42695,7 +43426,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [261 0 R 262 0 R 264 0 R 265 0 R]
@@ -43315,7 +44046,7 @@ endobj
/F1.0 10 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -43910,7 +44641,7 @@ endobj
/F4.0 35 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [285 0 R 286 0 R]
@@ -43930,7 +44661,7 @@ endobj
endobj
284 0 obj
<< /Limits [(hbase.cluster.distributed) (hbase.data.umask.enable)]
-/Names [(hbase.cluster.distributed) 200 0 R (hbase.column.max.version) 434 0 R (hbase.commit.msg.format) 3850 0 R (hbase.coordinated.state.manager.class) 477 0 R (hbase.coprocessor.abortonerror) 399 0 R (hbase.coprocessor.enabled) 392 0 R (hbase.coprocessor.master.classes) 398 0 R (hbase.coprocessor.region.classes) 397 0 R (hbase.coprocessor.user.enabled) 396 0 R (hbase.data.umask) 425 0 R (hbase.data.umask.enable) 424 0 R]
+/Names [(hbase.cluster.distributed) 200 0 R (hbase.column.max.version) 434 0 R (hbase.commit.msg.format) 3940 0 R (hbase.coordinated.state.manager.class) 477 0 R (hbase.coprocessor.abortonerror) 399 0 R (hbase.coprocessor.enabled) 392 0 R (hbase.coprocessor.master.classes) 398 0 R (hbase.coprocessor.region.classes) 397 0 R (hbase.coprocessor.user.enabled) 396 0 R (hbase.data.umask) 425 0 R (hbase.data.umask.enable) 424 0 R]
>>
endobj
285 0 obj
@@ -44476,7 +45207,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -44967,7 +45698,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -45539,7 +46270,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -46100,7 +46831,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -46697,7 +47428,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -47301,7 +48032,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -47844,7 +48575,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -48352,7 +49083,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -48927,7 +49658,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [350 0 R 354 0 R]
@@ -49496,7 +50227,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -49506,7 +50237,7 @@ endobj
endobj
359 0 obj
<< /Limits [(quota) (regions.arch)]
-/Names [(quota) 3224 0 R (read-api-and-usage) 1940 0 R (read-hbase-shell-commands-from-a-command-file) 788 0 R (reading-filtering-and-sending-edits) 3204 0 R (reading_cells_with_labels) 1412 0 R (recommended.configurations.hdfs) 546 0 R (recommended_configurations) 541 0 R (recommended_configurations.zk) 542 0 R (region-overlap-repairs) 3963 0 R (region-replication-for-meta-table-s-region) 1917 0 R (regions.arch) 1713 0 R]
+/Names [(quota) 3224 0 R (read-api-and-usage) 1940 0 R (read-hbase-shell-commands-from-a-command-file) 788 0 R (reading-filtering-and-sending-edits) 3204 0 R (reading_cells_with_labels) 1412 0 R (recommended.configurations.hdfs) 546 0 R (recommended_configurations) 541 0 R (recommended_configurations.zk) 542 0 R (references) 3828 0 R (region-overlap-repairs) 4052 0 R (region-replication-for-meta-table-s-region) 1917 0 R (region-transition-rpc-and-rs-heartbeat-can-arrive-at-same-time-on-master) 3861 0 R (regions.arch) 1713 0 R]
>>
endobj
360 0 obj
@@ -50107,7 +50838,7 @@ endobj
/F4.0 35 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [370 0 R 371 0 R 373 0 R 375 0 R 376 0 R]
@@ -50698,7 +51429,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -51240,7 +51971,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -51791,7 +52522,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -52310,7 +53041,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -52828,7 +53559,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -52838,7 +53569,7 @@ endobj
endobj
412 0 obj
<< /Limits [(hbase.zookeeper.property.maxClientCnxns) (hfile.block.index.cacheonwrite)]
-/Names [(hbase.zookeeper.property.maxClientCnxns) 273 0 R (hbase.zookeeper.property.syncLimit) 267 0 R (hbase.zookeeper.quorum) 203 0 R (hbase_apis) 2143 0 R (hbase_default_configurations) 197 0 R (hbase_env) 530 0 R (hbase_metrics) 3117 0 R (hbase_mob) 1954 0 R (hbase_site) 526 0 R (hbase_supported_tested_definitions) 42 0 R (hbck) 3008 0 R (hbck.in.depth) 3955 0 R (health.check) 3004 0 R (hedged.reads) 2607 0 R (hfile) 1770 0 R (hfile-format) 1771 0 R (hfile-format-2) 4058 0 R (hfile.block.bloom.cacheonwrite) 355 0 R (hfile.block.cache.size) 344 0 R (hfile.block.index.cacheonwrite) 345 0 R]
+/Names [(hbase.zookeeper.property.maxClientCnxns) 273 0 R (hbase.zookeeper.property.syncLimit) 267 0 R (hbase.zookeeper.quorum) 203 0 R (hbase_apis) 2143 0 R (hbase_default_configurations) 197 0 R (hbase_env) 530 0 R (hbase_metrics) 3117 0 R (hbase_mob) 1954 0 R (hbase_site) 526 0 R (hbase_supported_tested_definitions) 42 0 R (hbck) 3008 0 R (hbck.in.depth) 4044 0 R (health.check) 3004 0 R (hedged.reads) 2607 0 R (hfile) 1770 0 R (hfile-format) 1771 0 R (hfile-format-2) 4147 0 R (hfile.block.bloom.cacheonwrite) 355 0 R (hfile.block.cache.size) 344 0 R (hfile.block.index.cacheonwrite) 345 0 R]
>>
endobj
413 0 obj
@@ -53324,7 +54055,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -53337,7 +54068,7 @@ endobj
endobj
422 0 obj
<< /Limits [(hbase.rpc) (hbase.secure.spnego.ui)]
-/Names [(hbase.rpc) 4175 0 R (hbase.rpc.rows.warning.threshold) 509 0 R (hbase.rpc.shortoperation.timeout) 367 0 R (hbase.rpc.timeout) 361 0 R (hbase.rs.cacheblocksonwrite) 360 0 R (hbase.secure.bulkload) 1438 0 R (hbase.secure.configuration) 1278 0 R (hbase.secure.enable) 1442 0 R (hbase.secure.simpleconfiguration) 1320 0 R (hbase.secure.spnego.ui) 1273 0 R]
+/Names [(hbase.rpc) 4263 0 R (hbase.rpc.rows.warning.threshold) 509 0 R (hbase.rpc.shortoperation.timeout) 367 0 R (hbase.rpc.timeout) 361 0 R (hbase.rs.cacheblocksonwrite) 360 0 R (hbase.secure.bulkload) 1438 0 R (hbase.secure.configuration) 1278 0 R (hbase.secure.enable) 1442 0 R (hbase.secure.simpleconfiguration) 1320 0 R (hbase.secure.spnego.ui) 1273 0 R]
>>
endobj
423 0 obj
@@ -53858,7 +54589,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -54426,7 +55157,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -54943,7 +55674,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -55486,7 +56217,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [461 0 R 462 0 R]
@@ -56133,7 +56864,7 @@ endobj
/F4.0 35 0 R
/F6.0 471 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [467 0 R 468 0 R]
@@ -56174,11 +56905,11 @@ endobj
<< /Type /Font
/BaseFont /066905+mplus1mn-bold
/Subtype /TrueType
-/FontDescriptor 4909 0 R
+/FontDescriptor 5053 0 R
/FirstChar 32
/LastChar 255
-/Widths 4911 0 R
-/ToUnicode 4910 0 R
+/Widths 5055 0 R
+/ToUnicode 5054 0 R
>>
endobj
472 0 obj
@@ -56744,7 +57475,7 @@ endobj
/F3.0 33 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -56757,7 +57488,7 @@ endobj
endobj
478 0 obj
<< /Limits [(hbase.defaults.for.version.skip) (hbase.hregion.percolumnfamilyflush.size.lower.bound.min)]
-/Names [(hbase.defaults.for.version.skip) 407 0 R (hbase.dfs.client.read.shortcircuit.buffer.size) 439 0 R (hbase.display.keys) 391 0 R (hbase.dynamic.jars.dir) 456 0 R (hbase.encryption.server) 1425 0 R (hbase.env.sh) 511 0 R (hbase.fix.version.in.jira) 3838 0 R (hbase.history) 4135 0 R (hbase.hregion.majorcompaction) 314 0 R (hbase.hregion.majorcompaction.jitter) 315 0 R (hbase.hregion.max.filesize) 313 0 R (hbase.hregion.memstore.block.multiplier) 309 0 R (hbase.hregion.memstore.flush.size) 306 0 R (hbase.hregion.memstore.mslab.enabled) 310 0 R (hbase.hregion.percolumnfamilyflush.size.lower.bound.min) 307 0 R]
+/Names [(hbase.defaults.for.version.skip) 407 0 R (hbase.dfs.client.read.shortcircuit.buffer.size) 439 0 R (hbase.display.keys) 391 0 R (hbase.dynamic.jars.dir) 456 0 R (hbase.encryption.server) 1425 0 R (hbase.env.sh) 511 0 R (hbase.fix.version.in.jira) 3928 0 R (hbase.history) 4223 0 R (hbase.hregion.majorcompaction) 314 0 R (hbase.hregion.majorcompaction.jitter) 315 0 R (hbase.hregion.max.filesize) 313 0 R (hbase.hregion.memstore.block.multiplier) 309 0 R (hbase.hregion.memstore.flush.size) 306 0 R (hbase.hregion.memstore.mslab.enabled) 310 0 R (hbase.hregion.percolumnfamilyflush.size.lower.bound.min) 307 0 R]
>>
endobj
479 0 obj
@@ -57296,7 +58027,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -57811,7 +58542,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -58322,7 +59053,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -58853,7 +59584,7 @@ endobj
/F1.0 10 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -59964,7 +60695,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [517 0 R]
@@ -60812,7 +61543,7 @@ endobj
/F4.0 35 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [520 0 R]
@@ -62269,7 +63000,7 @@ endobj
/F3.0 33 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -62793,7 +63524,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -63446,7 +64177,7 @@ endobj
/F4.0 35 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [535 0 R 536 0 R 538 0 R 539 0 R 540 0 R 545 0 R]
@@ -64201,7 +64932,7 @@ endobj
/F2.0 29 0 R
/F1.1 38 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [551 0 R 554 0 R 557 0 R]
@@ -64240,7 +64971,7 @@ endobj
endobj
556 0 obj
<< /Limits [(configuration) (coprocessor-implementation-overview)]
-/Names [(configuration) 104 0 R (configuration-2) 3146 0 R (configuration-3) 3926 0 R (configuration-files) 111 0 R (configuration-from-scratch) 3823 0 R (configuration-properties) 1922 0 R (configure-mob-compaction-mergeable-threshold) 1963 0 R (configure-mob-compaction-policy) 1962 0 R (configuring-columns-for-mob) 1959 0 R (configuring-server-wide-behavior-of-bloom-filters) 2513 0 R (configuring-the-rest-server-and-client) 2162 0 R (confirm) 190 0 R (connection-setup) 4187 0 R (constraints) 1077 0 R (contributing-to-documentation-or-other-strings) 3861 0 R (coprocessor-implementation-overview) 2318 0 R]
+/Names [(configuration) 104 0 R (configuration-2) 3146 0 R (configuration-3) 4015 0 R (configuration-files) 111 0 R (configuration-from-scratch) 3913 0 R (configuration-properties) 1922 0 R (configure-mob-compaction-mergeable-threshold) 1963 0 R (configure-mob-compaction-policy) 1962 0 R (configuring-columns-for-mob) 1959 0 R (configuring-server-wide-behavior-of-bloom-filters) 2513 0 R (configuring-the-rest-server-and-client) 2162 0 R (confirm) 190 0 R (connection-setup) 4275 0 R (constraints) 1077 0 R (contributing-to-documentation-or-other-strings) 3951 0 R (coprocessor-implementation-overview) 2318 0 R]
>>
endobj
557 0 obj
@@ -65102,7 +65833,7 @@ endobj
/F4.0 35 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [561 0 R]
@@ -65916,7 +66647,7 @@ endobj
/F5.1 45 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [565 0 R 566 0 R 567 0 R 571 0 R 574 0 R 575 0 R 576 0 R]
@@ -66658,7 +67389,7 @@ endobj
/F1.1 38 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [580 0 R 581 0 R]
@@ -68090,7 +68821,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [585 0 R 586 0 R]
@@ -68867,7 +69598,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -69568,7 +70299,7 @@ endobj
/F3.0 33 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -70899,7 +71630,7 @@ endobj
/F4.0 35 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -72394,7 +73125,7 @@ endobj
/Font << /F2.0 29 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
>>
@@ -72560,7 +73291,7 @@ endobj
/Font << /F2.0 29 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -72758,7 +73489,7 @@ endobj
/Font << /F2.0 29 0 R
/F1.0 10 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [601 0 R 602 0 R 603 0 R 604 0 R 605 0 R]
@@ -73512,7 +74243,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [610 0 R]
@@ -74429,7 +75160,7 @@ endobj
/F3.0 33 0 R
/F5.1 45 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [614 0 R]
@@ -76744,7 +77475,7 @@ endobj
/F4.0 35 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [617 0 R 618 0 R]
@@ -77777,7 +78508,7 @@ endobj
/F1.1 38 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [622 0 R 623 0 R 624 0 R]
@@ -78441,7 +79172,7 @@ endobj
/F2.0 29 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [632 0 R 633 0 R 634 0 R 636 0 R 637 0 R 638 0 R]
@@ -79236,7 +79967,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [646 0 R]
@@ -79247,17 +79978,17 @@ endobj
endobj
642 0 obj
<< /Limits [(standalone.over.hdfs) (storefile-refresher)]
-/Names [(standalone.over.hdfs) 169 0 R (standalone_dist) 164 0 R (starting-and-stopping-the-rest-server) 2161 0 R (static-loading) 2352 0 R (static-unloading) 2355 0 R (store) 1764 0 R (store-file-ttl) 1916 0 R (store.file.dir) 1779 0 R (store.memstore) 1765 0 R (storefile-refresher) 1912 0 R]
+/Names [(standalone.over.hdfs) 169 0 R (standalone_dist) 164 0 R (starting-and-stopping-the-rest-server) 2161 0 R (statemachineprocedure) 3824 0 R (static-loading) 2352 0 R (static-unloading) 2355 0 R (store) 1764 0 R (store-file-ttl) 1916 0 R (store.file.dir) 1779 0 R (store.memstore) 1765 0 R (storefile-refresher) 1912 0 R]
>>
endobj
643 0 obj
<< /Limits [(__anchor-top) (cascading)]
-/Kids [157 0 R 3665 0 R 1881 0 R 1462 0 R 3903 0 R 1625 0 R 4084 0 R 2111 0 R 2069 0 R 2026 0 R 2060 0 R 3470 0 R]
+/Kids [157 0 R 3665 0 R 1881 0 R 1462 0 R 3800 0 R 1625 0 R 4172 0 R 2111 0 R 2069 0 R 2026 0 R 2060 0 R 3470 0 R]
>>
endobj
644 0 obj
<< /Limits [(hbase.mob.file.cache.size) (hbase.zookeeper.property.initLimit)]
-/Kids [501 0 R 4031 0 R 441 0 R 239 0 R 368 0 R 422 0 R 4176 0 R 455 0 R 268 0 R 3588 0 R 3559 0 R]
+/Kids [501 0 R 4120 0 R 441 0 R 239 0 R 368 0 R 422 0 R 4264 0 R 455 0 R 268 0 R 3588 0 R 3559 0 R]
>>
endobj
645 0 obj
@@ -79921,7 +80652,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -80636,7 +81367,7 @@ endobj
/F1.0 10 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [654 0 R]
@@ -81103,7 +81834,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -81823,7 +82554,7 @@ endobj
/F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [663 0 R 666 0 R]
@@ -83089,7 +83820,7 @@ endobj
/F3.0 33 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [669 0 R 670 0 R 671 0 R 672 0 R 673 0 R 674 0 R 675 0 R 676 0 R 679 0 R 680 0 R 681 0 R]
@@ -84025,7 +84756,7 @@ endobj
/Font << /F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [684 0 R 685 0 R 688 0 R]
@@ -84924,7 +85655,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [693 0 R 694 0 R 696 0 R 698 0 R 699 0 R]
@@ -85743,7 +86474,7 @@ endobj
/Font << /F1.0 10 0 R
/F3.0 33 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [702 0 R 705 0 R 708 0 R 709 0 R]
@@ -86747,7 +87478,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [714 0 R 716 0 R 718 0 R]
@@ -87567,7 +88298,7 @@ endobj
/F3.0 33 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [723 0 R 729 0 R 730 0 R 731 0 R]
@@ -88604,7 +89335,7 @@ endobj
/F2.0 29 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [737 0 R 738 0 R 739 0 R 741 0 R 742 0 R 743 0 R 744 0 R 745 0 R 746 0 R 747 0 R 748 0 R 749 0 R 750 0 R 751 0 R 752 0 R]
@@ -89379,7 +90110,7 @@ endobj
/Font << /F1.0 10 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [755 0 R 756 0 R]
@@ -89685,7 +90416,7 @@ endobj
/F1.0 10 0 R
/F2.0 29 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -90039,7 +90770,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [769 0 R 770 0 R 771 0 R]
@@ -90240,7 +90971,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
>>
@@ -90478,7 +91209,7 @@ endobj
/F1.0 10 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp2 4655 0 R
+/XObject << /Stamp2 4781 0 R
>>
>>
/Annots [778 0 R]
@@ -91074,7 +91805,7 @@ endobj
/F3.0 33 0 R
/F4.0 35 0 R
>>
-/XObject << /Stamp1 4654 0 R
+/XObject << /Stamp1 4780 0 R
>>
>>
/Annots [782 0 R]
@@ -91611,7 +92342,7
<TRUNCATED>
[18/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
index f308f4e..cd9dbb3 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public abstract static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1081">WALSplitter.OutputSink</a>
+<pre>public abstract static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1082">WALSplitter.OutputSink</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">The following class is an abstraction class to provide a common interface to support
different ways of consuming recovered edits.</div>
@@ -307,7 +307,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>controller</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1083">controller</a></pre>
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1084">controller</a></pre>
</li>
</ul>
<a name="entryBuffers">
@@ -316,7 +316,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>entryBuffers</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1084">entryBuffers</a></pre>
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1085">entryBuffers</a></pre>
</li>
</ul>
<a name="writers">
@@ -325,7 +325,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>writers</h4>
-<pre>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.SinkWriter</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1086">writers</a></pre>
+<pre>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.SinkWriter</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1087">writers</a></pre>
</li>
</ul>
<a name="regionMaximumEditLogSeqNum">
@@ -334,7 +334,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>regionMaximumEditLogSeqNum</h4>
-<pre>protected final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1087">regionMaximumEditLogSeqNum</a></pre>
+<pre>protected final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1088">regionMaximumEditLogSeqNum</a></pre>
</li>
</ul>
<a name="writerThreads">
@@ -343,7 +343,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>writerThreads</h4>
-<pre>protected final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterThread</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1091">writerThreads</a></pre>
+<pre>protected final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterThread</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1092">writerThreads</a></pre>
</li>
</ul>
<a name="blacklistedRegions">
@@ -352,7 +352,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>blacklistedRegions</h4>
-<pre>protected final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><byte[]> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1094">blacklistedRegions</a></pre>
+<pre>protected final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><byte[]> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1095">blacklistedRegions</a></pre>
</li>
</ul>
<a name="closeAndCleanCompleted">
@@ -361,7 +361,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>closeAndCleanCompleted</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1097">closeAndCleanCompleted</a></pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1098">closeAndCleanCompleted</a></pre>
</li>
</ul>
<a name="writersClosed">
@@ -370,7 +370,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>writersClosed</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1099">writersClosed</a></pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1100">writersClosed</a></pre>
</li>
</ul>
<a name="numThreads">
@@ -379,7 +379,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>numThreads</h4>
-<pre>protected final int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1101">numThreads</a></pre>
+<pre>protected final int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1102">numThreads</a></pre>
</li>
</ul>
<a name="reporter">
@@ -388,7 +388,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>reporter</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/util/CancelableProgressable.html" title="interface in org.apache.hadoop.hbase.util">CancelableProgressable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1103">reporter</a></pre>
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/util/CancelableProgressable.html" title="interface in org.apache.hadoop.hbase.util">CancelableProgressable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1104">reporter</a></pre>
</li>
</ul>
<a name="skippedEdits">
@@ -397,7 +397,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>skippedEdits</h4>
-<pre>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1105">skippedEdits</a></pre>
+<pre>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1106">skippedEdits</a></pre>
</li>
</ul>
<a name="splits">
@@ -406,7 +406,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>splits</h4>
-<pre>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1107">splits</a></pre>
+<pre>protected <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1108">splits</a></pre>
</li>
</ul>
</li>
@@ -423,7 +423,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>OutputSink</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1109">OutputSink</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1110">OutputSink</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> entryBuffers,
int numWriters)</pre>
</li>
@@ -442,7 +442,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setReporter</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1115">setReporter</a>(<a href="../../../../../org/apache/hadoop/hbase/util/CancelableProgressable.html" title="interface in org.apache.hadoop.hbase.util">CancelableProgressable</a> reporter)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1116">setReporter</a>(<a href="../../../../../org/apache/hadoop/hbase/util/CancelableProgressable.html" title="interface in org.apache.hadoop.hbase.util">CancelableProgressable</a> reporter)</pre>
</li>
</ul>
<a name="startWriterThreads--">
@@ -451,7 +451,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>startWriterThreads</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1122">startWriterThreads</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1123">startWriterThreads</a>()</pre>
<div class="block">Start the threads that will pump data from the entryBuffers to the output files.</div>
</li>
</ul>
@@ -461,7 +461,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>updateRegionMaximumEditLogSeqNum</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1134">updateRegionMaximumEditLogSeqNum</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1135">updateRegionMaximumEditLogSeqNum</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
<div class="block">Update region's maximum edit log SeqNum.</div>
</li>
</ul>
@@ -471,7 +471,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getNumOpenWriters</h4>
-<pre>int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1147">getNumOpenWriters</a>()</pre>
+<pre>int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1148">getNumOpenWriters</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the number of currently opened writers</dd>
@@ -484,7 +484,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getSkippedEdits</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1151">getSkippedEdits</a>()</pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1152">getSkippedEdits</a>()</pre>
</li>
</ul>
<a name="finishWriting-boolean-">
@@ -493,7 +493,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>finishWriting</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1160">finishWriting</a>(boolean interrupt)
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1161">finishWriting</a>(boolean interrupt)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Wait for writer threads to dump all info to the sink</div>
<dl>
@@ -510,7 +510,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>finishWritingAndClose</h4>
-<pre>public abstract <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1189">finishWritingAndClose</a>()
+<pre>public abstract <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1190">finishWritingAndClose</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -524,7 +524,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getOutputCounts</h4>
-<pre>public abstract <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1194">getOutputCounts</a>()</pre>
+<pre>public abstract <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1195">getOutputCounts</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>a map from encoded region ID to the number of edits written out for that region.</dd>
@@ -537,7 +537,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getNumberOfRecoveredRegions</h4>
-<pre>public abstract int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1199">getNumberOfRecoveredRegions</a>()</pre>
+<pre>public abstract int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1200">getNumberOfRecoveredRegions</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>number of regions we've recovered</dd>
@@ -550,7 +550,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>append</h4>
-<pre>public abstract void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1205">append</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
+<pre>public abstract void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1206">append</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -566,7 +566,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>flush</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1211">flush</a>()
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1212">flush</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">WriterThread call this function to help flush internal remaining edits in buffer before close</div>
<dl>
@@ -583,7 +583,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>keepRegionEvent</h4>
-<pre>public abstract boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1221">keepRegionEvent</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
+<pre>public abstract boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#line.1222">keepRegionEvent</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
<div class="block">Some WALEdit's contain only KV's for account on what happened to a region.
Not all sinks will want to get all of those edits.</div>
<dl>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
index e01215f..2682dc7 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
@@ -113,7 +113,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.805">WALSplitter.PipelineController</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.806">WALSplitter.PipelineController</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">Contains some methods to control WAL-entries producer / consumer interactions</div>
</li>
@@ -212,7 +212,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>thrown</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicReference.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicReference</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.808">thrown</a></pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicReference.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicReference</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.809">thrown</a></pre>
</li>
</ul>
<a name="dataAvailable">
@@ -221,7 +221,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>dataAvailable</h4>
-<pre>public final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.812">dataAvailable</a></pre>
+<pre>public final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.813">dataAvailable</a></pre>
</li>
</ul>
</li>
@@ -238,7 +238,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>PipelineController</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.805">PipelineController</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.806">PipelineController</a>()</pre>
</li>
</ul>
</li>
@@ -255,7 +255,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>writerThreadError</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.814">writerThreadError</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> t)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.815">writerThreadError</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> t)</pre>
</li>
</ul>
<a name="checkForErrors--">
@@ -264,7 +264,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>checkForErrors</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.821">checkForErrors</a>()
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html#line.822">checkForErrors</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Check for errors in the writer threads. If any is found, rethrow it.</div>
<dl>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
index b42fd4e..cb17383 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.966">WALSplitter.RegionEntryBuffer</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.967">WALSplitter.RegionEntryBuffer</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" title="interface in org.apache.hadoop.hbase.io">HeapSize</a></pre>
<div class="block">A buffer of some number of edits for a given region.
@@ -243,7 +243,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>heapInBuffer</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.967">heapInBuffer</a></pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.968">heapInBuffer</a></pre>
</li>
</ul>
<a name="entryBuffer">
@@ -252,7 +252,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>entryBuffer</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.968">entryBuffer</a></pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.969">entryBuffer</a></pre>
</li>
</ul>
<a name="tableName">
@@ -261,7 +261,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>tableName</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.969">tableName</a></pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.970">tableName</a></pre>
</li>
</ul>
<a name="encodedRegionName">
@@ -270,7 +270,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockListLast">
<li class="blockList">
<h4>encodedRegionName</h4>
-<pre>byte[] <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.970">encodedRegionName</a></pre>
+<pre>byte[] <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.971">encodedRegionName</a></pre>
</li>
</ul>
</li>
@@ -287,7 +287,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockListLast">
<li class="blockList">
<h4>RegionEntryBuffer</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.972">RegionEntryBuffer</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> tableName,
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.973">RegionEntryBuffer</a>(<a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> tableName,
byte[] region)</pre>
</li>
</ul>
@@ -305,7 +305,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>appendEntry</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.978">appendEntry</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.979">appendEntry</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
</li>
</ul>
<a name="internify-org.apache.hadoop.hbase.wal.WAL.Entry-">
@@ -314,7 +314,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>internify</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.988">internify</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.989">internify</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
</li>
</ul>
<a name="heapSize--">
@@ -323,7 +323,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>heapSize</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.995">heapSize</a>()</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.996">heapSize</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html#heapSize--">heapSize</a></code> in interface <code><a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" title="interface in org.apache.hadoop.hbase.io">HeapSize</a></code></dd>
@@ -339,7 +339,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>getEncodedRegionName</h4>
-<pre>public byte[] <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.999">getEncodedRegionName</a>()</pre>
+<pre>public byte[] <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.1000">getEncodedRegionName</a>()</pre>
</li>
</ul>
<a name="getEntryBuffer--">
@@ -348,7 +348,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockList">
<li class="blockList">
<h4>getEntryBuffer</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.1003">getEntryBuffer</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.1004">getEntryBuffer</a>()</pre>
</li>
</ul>
<a name="getTableName--">
@@ -357,7 +357,7 @@ implements <a href="../../../../../org/apache/hadoop/hbase/io/HeapSize.html" tit
<ul class="blockListLast">
<li class="blockList">
<h4>getTableName</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.1007">getTableName</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/TableName.html" title="class in org.apache.hadoop.hbase">TableName</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html#line.1008">getTableName</a>()</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
index 0335c16..43eb417 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public abstract static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1725">WALSplitter.SinkWriter</a>
+<pre>public abstract static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1730">WALSplitter.SinkWriter</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">Class wraps the actual writer which writes data out and related statistics</div>
</li>
@@ -222,7 +222,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>editsWritten</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1727">editsWritten</a></pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1732">editsWritten</a></pre>
</li>
</ul>
<a name="editsSkipped">
@@ -231,7 +231,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>editsSkipped</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1729">editsSkipped</a></pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1734">editsSkipped</a></pre>
</li>
</ul>
<a name="nanosSpent">
@@ -240,7 +240,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>nanosSpent</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1731">nanosSpent</a></pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1736">nanosSpent</a></pre>
</li>
</ul>
</li>
@@ -257,7 +257,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>SinkWriter</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1725">SinkWriter</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1730">SinkWriter</a>()</pre>
</li>
</ul>
</li>
@@ -274,7 +274,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>incrementEdits</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1733">incrementEdits</a>(int edits)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1738">incrementEdits</a>(int edits)</pre>
</li>
</ul>
<a name="incrementSkippedEdits-int-">
@@ -283,7 +283,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>incrementSkippedEdits</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1737">incrementSkippedEdits</a>(int skipped)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1742">incrementSkippedEdits</a>(int skipped)</pre>
</li>
</ul>
<a name="incrementNanoTime-long-">
@@ -292,7 +292,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>incrementNanoTime</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1741">incrementNanoTime</a>(long nanos)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html#line.1746">incrementNanoTime</a>(long nanos)</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
index e5cd891..4344c1e 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
@@ -112,7 +112,7 @@
</dl>
<hr>
<br>
-<pre>private static final class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1750">WALSplitter.WriterAndPath</a>
+<pre>private static final class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1755">WALSplitter.WriterAndPath</a>
extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.SinkWriter</a></pre>
<div class="block">Private data structure that wraps a Writer and its Path, also collecting statistics about the
data written to this output.</div>
@@ -215,7 +215,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWrit
<ul class="blockList">
<li class="blockList">
<h4>p</h4>
-<pre>final org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1751">p</a></pre>
+<pre>final org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1756">p</a></pre>
</li>
</ul>
<a name="w">
@@ -224,7 +224,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWrit
<ul class="blockList">
<li class="blockList">
<h4>w</h4>
-<pre>final <a href="../../../../../org/apache/hadoop/hbase/wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1752">w</a></pre>
+<pre>final <a href="../../../../../org/apache/hadoop/hbase/wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1757">w</a></pre>
</li>
</ul>
<a name="minLogSeqNum">
@@ -233,7 +233,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWrit
<ul class="blockListLast">
<li class="blockList">
<h4>minLogSeqNum</h4>
-<pre>final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1753">minLogSeqNum</a></pre>
+<pre>final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1758">minLogSeqNum</a></pre>
</li>
</ul>
</li>
@@ -250,7 +250,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.SinkWrit
<ul class="blockListLast">
<li class="blockList">
<h4>WriterAndPath</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1755">WriterAndPath</a>(org.apache.hadoop.fs.Path p,
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html#line.1760">WriterAndPath</a>(org.apache.hadoop.fs.Path p,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a> w,
long minLogSeqNum)</pre>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
index 3e0f740..2856cae 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1012">WALSplitter.WriterThread</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1013">WALSplitter.WriterThread</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true" title="class or interface in java.lang">Thread</a></pre>
</li>
</ul>
@@ -266,7 +266,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>shouldStop</h4>
-<pre>private volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1013">shouldStop</a></pre>
+<pre>private volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1014">shouldStop</a></pre>
</li>
</ul>
<a name="controller">
@@ -275,7 +275,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>controller</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1014">controller</a></pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1015">controller</a></pre>
</li>
</ul>
<a name="entryBuffers">
@@ -284,7 +284,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>entryBuffers</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1015">entryBuffers</a></pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1016">entryBuffers</a></pre>
</li>
</ul>
<a name="outputSink">
@@ -293,7 +293,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockListLast">
<li class="blockList">
<h4>outputSink</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.OutputSink</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1016">outputSink</a></pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.OutputSink</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1017">outputSink</a></pre>
</li>
</ul>
</li>
@@ -310,7 +310,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockListLast">
<li class="blockList">
<h4>WriterThread</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1018">WriterThread</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1019">WriterThread</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> entryBuffers,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.OutputSink</a> sink,
int i)</pre>
@@ -330,7 +330,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>run</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1026">run</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1027">run</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true#run--" title="class or interface in java.lang">run</a></code> in interface <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true" title="class or interface in java.lang">Runnable</a></code></dd>
@@ -345,7 +345,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>doRun</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1035">doRun</a>()
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1036">doRun</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -359,7 +359,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockList">
<li class="blockList">
<h4>writeBuffer</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1065">writeBuffer</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1066">writeBuffer</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -373,7 +373,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html
<ul class="blockListLast">
<li class="blockList">
<h4>finish</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1069">finish</a>()</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html#line.1070">finish</a>()</pre>
</li>
</ul>
</li>
[20/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/book.html
----------------------------------------------------------------------
diff --git a/book.html b/book.html
index 03054ef..151f29b 100644
--- a/book.html
+++ b/book.html
@@ -294,17 +294,41 @@
<li><a href="#_protobuf">175. Protobuf</a></li>
</ul>
</li>
+<li><a href="#pv2">Procedure Framework (Pv2): <a href="https://issues.apache.org/jira/browse/HBASE-12439">HBASE-12439</a></a>
+<ul class="sectlevel1">
+<li><a href="#_procedures">176. Procedures</a></li>
+<li><a href="#_subprocedures">177. Subprocedures</a></li>
+<li><a href="#_procedureexecutor">178. ProcedureExecutor</a></li>
+<li><a href="#_nonces">179. Nonces</a></li>
+<li><a href="#_wait_wake_suspend_yield">180. Wait/Wake/Suspend/Yield</a></li>
+<li><a href="#_locking">181. Locking</a></li>
+<li><a href="#_procedure_types">182. Procedure Types</a></li>
+<li><a href="#_references">183. References</a></li>
+</ul>
+</li>
+<li><a href="#amv2">AMv2 Description for Devs</a>
+<ul class="sectlevel1">
+<li><a href="#_background">184. Background</a></li>
+<li><a href="#_new_system">185. New System</a></li>
+<li><a href="#_procedures_detail">186. Procedures Detail</a></li>
+<li><a href="#_ui">187. UI</a></li>
+<li><a href="#_logging">188. Logging</a></li>
+<li><a href="#_implementation_notes">189. Implementation Notes</a></li>
+<li><a href="#_new_configs">190. New Configs</a></li>
+<li><a href="#_tools">191. Tools</a></li>
+</ul>
+</li>
<li><a href="#zookeeper">ZooKeeper</a>
<ul class="sectlevel1">
-<li><a href="#_using_existing_zookeeper_ensemble">176. Using existing ZooKeeper ensemble</a></li>
-<li><a href="#zk.sasl.auth">177. SASL Authentication with ZooKeeper</a></li>
+<li><a href="#_using_existing_zookeeper_ensemble">192. Using existing ZooKeeper ensemble</a></li>
+<li><a href="#zk.sasl.auth">193. SASL Authentication with ZooKeeper</a></li>
</ul>
</li>
<li><a href="#community">Community</a>
<ul class="sectlevel1">
-<li><a href="#_decisions">178. Decisions</a></li>
-<li><a href="#community.roles">179. Community Roles</a></li>
-<li><a href="#hbase.commit.msg.format">180. Commit Message format</a></li>
+<li><a href="#_decisions">194. Decisions</a></li>
+<li><a href="#community.roles">195. Community Roles</a></li>
+<li><a href="#hbase.commit.msg.format">196. Commit Message format</a></li>
</ul>
</li>
<li><a href="#_appendix">Appendix</a>
@@ -322,8 +346,8 @@
<li><a href="#asf">Appendix K: HBase and the Apache Software Foundation</a></li>
<li><a href="#orca">Appendix L: Apache HBase Orca</a></li>
<li><a href="#tracing">Appendix M: Enabling Dapper-like Tracing in HBase</a></li>
-<li><a href="#tracing.client.modifications">181. Client Modifications</a></li>
-<li><a href="#tracing.client.shell">182. Tracing from HBase Shell</a></li>
+<li><a href="#tracing.client.modifications">197. Client Modifications</a></li>
+<li><a href="#tracing.client.shell">198. Tracing from HBase Shell</a></li>
<li><a href="#hbase.rpc">Appendix N: 0.95 RPC Specification</a></li>
</ul>
</li>
@@ -33910,6 +33934,509 @@ public API. TODO.</p>
</div>
</div>
</div>
+<h1 id="pv2" class="sect0"><a class="anchor" href="#pv2"></a>Procedure Framework (Pv2): <a href="https://issues.apache.org/jira/browse/HBASE-12439">HBASE-12439</a></h1>
+<div class="openblock partintro">
+<div class="content">
+<div class="paragraph">
+<p><em>Procedure v2 …​aims to provide a unified way to build…​multi-step procedures with a rollback/roll-forward ability in case of failure (e.g. create/delete table) — Matteo Bertozzi, the author of Pv2.</em></p>
+</div>
+<div class="paragraph">
+<p>With Pv2 you can build and run state machines. It was built by Matteo to make distributed state transitions in HBase resilient in the face of process failures. Previous to Pv2, state transition handling was spread about the codebase with implementation varying by transition-type and context. Pv2 was inspired by <a href="https://accumulo.apache.org/1.8/accumulo_user_manual.html#_fault_tolerant_executor_fate">FATE</a>, of Apache Accumulo.<br></p>
+</div>
+<div class="paragraph">
+<p>Early Pv2 aspects have been shipping in HBase with a good while now but it has continued to evolve as it takes on more involved scenarios. What we have now is powerful but intricate in operation and incomplete, in need of cleanup and hardening. In this doc we have given overview on the system so you can make use of it (and help with its polishing).</p>
+</div>
+<div class="paragraph">
+<p>This system has the awkward name of Pv2 because HBase already had the notion of a Procedure used in snapshots (see hbase-server <em>org.apache.hadoop.hbase.procedure</em> as opposed to hbase-procedure <em>org.apache.hadoop.hbase.procedure2</em>). Pv2 supercedes and is to replace Procedure.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_procedures"><a class="anchor" href="#_procedures"></a>176. Procedures</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>A Procedure is a transform made on an HBase entity. Examples of HBase entities would be Regions and Tables.<br>
+Procedures are run by a ProcedureExecutor instance. Procedure current state is kept in the ProcedureStore.<br>
+The ProcedureExecutor has but a primitive view on what goes on inside a Procedure. From its PoV, Procedures are submitted and then the ProcedureExecutor keeps calling <em>#execute(Object)</em> until the Procedure is done. Execute may be called multiple times in the case of failure or restart, so Procedure code must be idempotent yielding the same result each time it run. Procedure code can also implement <em>rollback</em> so steps can be undone if failure. A call to <em>execute()</em> can result in one of following possibilities:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><em>execute()</em> returns</p>
+<div class="ulist">
+<ul>
+<li>
+<p><em>null</em>: indicates we are done.</p>
+</li>
+<li>
+<p><em>this</em>: indicates there is more to do so, persist current procedure state and re-<em>execute()</em>.</p>
+</li>
+<li>
+<p><em>Array</em> of sub-procedures: indicates a set of procedures needed to be run to completion before we can proceed (after which we expect the framework to call our execute again).</p>
+</li>
+</ul>
+</div>
+</li>
+<li>
+<p><em>execute()</em> throws exception</p>
+<div class="ulist">
+<ul>
+<li>
+<p><em>suspend</em>: indicates execution of procedure is suspended and can be resumed due to some external event. The procedure state is persisted.</p>
+</li>
+<li>
+<p><em>yield</em>: procedure is added back to scheduler. The procedure state is not persisted.</p>
+</li>
+<li>
+<p><em>interrupted</em>: currently same as <em>yield</em>.</p>
+</li>
+<li>
+<p>Any <em>exception</em> not listed above: Procedure <em>state</em> is changed to <em>FAILED</em> (after which we expect the framework will attempt rollback).</p>
+</li>
+</ul>
+</div>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>The ProcedureExecutor stamps the frameworks notions of Procedure State into the Procedure itself; e.g. it marks Procedures as INITIALIZING on submit. It moves the state to RUNNABLE when it goes to execute. When done, a Procedure gets marked FAILED or SUCCESS depending. Here is the list of all states as of this writing:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><strong><em>INITIALIZING</em></strong> Procedure in construction, not yet added to the executor</p>
+</li>
+<li>
+<p><strong><em>RUNNABLE</em></strong> Procedure added to the executor, and ready to be executed.</p>
+</li>
+<li>
+<p><strong><em>WAITING</em></strong> The procedure is waiting on children (subprocedures) to be completed</p>
+</li>
+<li>
+<p><strong><em>WAITING_TIMEOUT</em></strong> The procedure is waiting a timeout or an external event</p>
+</li>
+<li>
+<p><strong><em>ROLLEDBACK</em></strong> The procedure failed and was rolledback.</p>
+</li>
+<li>
+<p><strong><em>SUCCESS</em></strong> The procedure execution completed successfully.</p>
+</li>
+<li>
+<p><strong><em>FAILED</em></strong> The procedure execution failed, may need to rollback.</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>After each execute, the Procedure state is persisted to the ProcedureStore. Hooks are invoked on Procedures so they can preserve custom state. Post-fault, the ProcedureExecutor re-hydrates its pre-crash state by replaying the content of the ProcedureStore. This makes the Procedure Framework resilient against process failure.</p>
+</div>
+<div class="sect2">
+<h3 id="_implementation"><a class="anchor" href="#_implementation"></a>176.1. Implementation</h3>
+<div class="paragraph">
+<p>In implementation, Procedures tend to divide transforms into finer-grained tasks and while some of these work items are handed off to sub-procedures,
+the bulk are done as processing <em>steps</em> in-Procedure; each invocation of the execute is used to perform a single step, and then the Procedure relinquishes returning to the framework. The Procedure does its own tracking of where it is in the processing.</p>
+</div>
+<div class="paragraph">
+<p>What comprises a sub-task, or <em>step</em> in the execution is up to the Procedure author but generally it is a small piece of work that cannot be further decomposed and that moves the processing forward toward its end state. Having procedures made of many small steps rather than a few large ones allows the Procedure framework give out insight on where we are in the processing. It also allows the framework be more fair in its execution. As stated per above, each step may be called multiple times (failure/restart) so steps must be implemented idempotent.<br>
+It is easy to confuse the state that the Procedure itself is keeping with that of the Framework itself. Try to keep them distinct.<br></p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_rollback_2"><a class="anchor" href="#_rollback_2"></a>176.2. Rollback</h3>
+<div class="paragraph">
+<p>Rollback is called when the procedure or one of the sub-procedures has failed. The rollback step is supposed to cleanup the resources created during the execute() step. In case of failure and restart, rollback() may be called multiple times, so again the code must be idempotent.</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_metrics_2"><a class="anchor" href="#_metrics_2"></a>176.3. Metrics</h3>
+<div class="paragraph">
+<p>There are hooks for collecting metrics on submit of the procedure and on finish.</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p>updateMetricsOnSubmit()</p>
+</li>
+<li>
+<p>updateMetricsOnFinish()</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>Individual procedures can override these methods to collect procedure specific metrics. The default implementations of these methods try to get an object implementing an interface ProcedureMetrics which encapsulates following set of generic metrics:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p>SubmittedCount (Counter): Total number of procedure instances submitted of a type.</p>
+</li>
+<li>
+<p>Time (Histogram): Histogram of runtime for procedure instances.</p>
+</li>
+<li>
+<p>FailedCount (Counter): Total number of failed procedure instances.</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>Individual procedures can implement this object and define these generic set of metrics.</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_baggage"><a class="anchor" href="#_baggage"></a>176.4. Baggage</h3>
+<div class="paragraph">
+<p>Procedures can carry baggage. One example is the <em>step</em> the procedure last attained (see previous section); procedures persist the enum that marks where they are currently. Other examples might be the Region or Server name the Procedure is currently working. After each call to execute, the Procedure#serializeStateData is called. Procedures can persist whatever.</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_result_state_and_queries"><a class="anchor" href="#_result_state_and_queries"></a>176.5. Result/State and Queries</h3>
+<div class="paragraph">
+<p>(From Matteo’s <a href="https://issues.apache.org/jira/secure/attachment/12693273/Procedurev2Notification-Bus.pdf">ProcedureV2 and Notification Bus</a> doc)<br>
+In the case of asynchronous operations, the result must be kept around until the client asks for it. Once we receive a “get” of the result we can schedule the delete of the record. For some operations the result may be “unnecessary” especially in case of failure (e.g. if the create table fail, we can query the operation result or we can just do a list table to see if it was created) so in some cases we can schedule the delete after a timeout. On the client side the operation will return a “Procedure ID”, this ID can be used to wait until the procedure is completed and get the result/exception.<br></p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java">Admin.doOperation() { longprocId=master.doOperation(); master.waitCompletion(procId); } +</code></pre>
+</div>
+</div>
+<div class="paragraph">
+<p>If the master goes down while performing the operation the backup master will pickup the half inprogress operation and complete it. The client will not notice the failure.</p>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_subprocedures"><a class="anchor" href="#_subprocedures"></a>177. Subprocedures</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>Subprocedures are <em>Procedure</em> instances created and returned by <em>#execute(Object)</em> method of a procedure instance (parent procedure). As subprocedures are of type <em>Procedure</em>, they can instantiate their own subprocedures. As its a recursive, procedure stack is maintained by the framework. The framework makes sure that the parent procedure does not proceed till all sub-procedures and their subprocedures in a procedure stack are successfully finished.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_procedureexecutor"><a class="anchor" href="#_procedureexecutor"></a>178. ProcedureExecutor</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p><em>ProcedureExecutor</em> uses <em>ProcedureStore</em> and <em>ProcedureScheduler</em> and executes procedures submitted to it. Some of the basic operations supported are:</p>
+</div>
+<div class="ulist">
+<ul>
+<li>
+<p><em>abort(procId)</em>: aborts specified procedure if its not finished</p>
+</li>
+<li>
+<p><em>submit(Procedure)</em>: submits procedure for execution</p>
+</li>
+<li>
+<p><em>retrieve:</em> list of get methods to get <em>Procedure</em> instances and results</p>
+</li>
+<li>
+<p><em>register/ unregister</em> listeners: for listening on Procedure related notifications</p>
+</li>
+</ul>
+</div>
+<div class="paragraph">
+<p>When <em>ProcedureExecutor</em> starts it loads procedure instances persisted in <em>ProcedureStore</em> from previous run. All unfinished procedures are resumed from the last stored state.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_nonces"><a class="anchor" href="#_nonces"></a>179. Nonces</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>You can pass the nonce that came in with the RPC to the Procedure on submit at the executor. This nonce will then be serialized along w/ the Procedure on persist. If a crash, on reload, the nonce will be put back into a map of nonces to pid in case a client tries to run same procedure for a second time (it will be rejected). See the base Procedure and how nonce is a base data member.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_wait_wake_suspend_yield"><a class="anchor" href="#_wait_wake_suspend_yield"></a>180. Wait/Wake/Suspend/Yield</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>‘suspend’ means stop processing a procedure because we can make no more progress until a condition changes; i.e. we sent RPC and need to wait on response. The way this works is that a Procedure throws a suspend exception from down in its guts as a GOTO the end-of-the-current-processing step. Suspend also puts the Procedure back on the scheduler. Problematic is we do some accounting on our way out even on suspend making it so it can take time exiting (We have to update state in the WAL).</p>
+</div>
+<div class="paragraph">
+<p>RegionTransitionProcedure#reportTransition is called on receipt of a report from a RS. For Assign and Unassign, this event response from the server we sent an RPC wakes up suspended Assign/Unassigns.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_locking"><a class="anchor" href="#_locking"></a>181. Locking</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>Procedure Locks are not about concurrency! They are about giving a Procedure read/write access to an HBase Entity such as a Table or Region so that is possible to shut out other Procedures from making modifications to an HBase Entity state while the current one is running.</p>
+</div>
+<div class="paragraph">
+<p>Locking is optional, up to the Procedure implementor but if an entity is being operated on by a Procedure, all transforms need to be done via Procedures using the same locking scheme else havoc.</p>
+</div>
+<div class="paragraph">
+<p>Two ProcedureExecutor Worker threads can actually end up both processing the same Procedure instance. If it happens, the threads are meant to be running different parts of the one Procedure — changes that do not stamp on each other (This gets awkward around the procedure frameworks notion of ‘suspend’. More on this below).</p>
+</div>
+<div class="paragraph">
+<p>Locks optionally may be held for the life of a Procedure. For example, if moving a Region, you probably want to have exclusive access to the HBase Region until the Region completes (or fails). This is used in conjunction with {@link #holdLock(Object)}. If {@link #holdLock(Object)} returns true, the procedure executor will call acquireLock() once and thereafter not call {@link #releaseLock(Object)} until the Procedure is done (Normally, it calls release/acquire around each invocation of {@link #execute(Object)}.</p>
+</div>
+<div class="paragraph">
+<p>Locks also may live the life of a procedure; i.e. once an Assign Procedure starts, we do not want another procedure meddling w/ the region under assignment. Procedures that hold the lock for the life of the procedure set Procedure#holdLock to true. AssignProcedure does this as do Split and Move (If in the middle of a Region move, you do not want it Splitting).</p>
+</div>
+<div class="paragraph">
+<p>Locking can be for life of Procedure.</p>
+</div>
+<div class="paragraph">
+<p>Some locks have a hierarchy. For example, taking a region lock also takes (read) lock on its containing table and namespace to prevent another Procedure obtaining an exclusive lock on the hosting table (or namespace).</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_procedure_types"><a class="anchor" href="#_procedure_types"></a>182. Procedure Types</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_statemachineprocedure"><a class="anchor" href="#_statemachineprocedure"></a>182.1. StateMachineProcedure</h3>
+<div class="paragraph">
+<p>One can consider each call to <em>#execute(Object)</em> method as transitioning from one state to another in a state machine. Abstract class <em>StateMachineProcedure</em> is wrapper around base <em>Procedure</em> class which provides constructs for implementing a state machine as a <em>Procedure</em>. After each state transition current state is persisted so that, in case of crash/ restart, the state transition can be resumed from the previous state of a procedure before crash/ restart. Individual procedures need to define initial and terminus states and hooks <em>executeFromState()</em> and <em>setNextState()</em> are provided for state transitions.</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_remoteproceduredispatcher"><a class="anchor" href="#_remoteproceduredispatcher"></a>182.2. RemoteProcedureDispatcher</h3>
+<div class="paragraph">
+<p>A new RemoteProcedureDispatcher (+ subclass RSProcedureDispatcher) primitive takes care of running the Procedure-based Assignments ‘remote’ component. This dispatcher knows about ‘servers’. It does aggregation of assignments by time on a time/count basis so can send procedures in batches rather than one per RPC. Procedure status comes back on the back of the RegionServer heartbeat reporting online/offline regions (No more notifications via ZK). The response is passed to the AMv2 to ‘process’. It will check against the in-memory state. If there is a mismatch, it fences out the RegionServer on the assumption that something went wrong on the RS side. Timeouts trigger retries (Not Yet Implemented!). The Procedure machine ensures only one operation at a time on any one Region/Table using entity <em>locking</em> and smarts about what is serial and what can be run concurrently (Locking was zk-based — you’d put a znode in zk for a table — 
;but now has been converted to be procedure-based as part of this project).</p>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_references"><a class="anchor" href="#_references"></a>183. References</h2>
+<div class="sectionbody">
+<div class="ulist">
+<ul>
+<li>
+<p>Matteo had a slide deck on what it the Procedure Framework would look like and the problems it addresses initially <a href="https://issues.apache.org/jira/secure/attachment/12845124/ProcedureV2b.pdf">attached to the Pv2 issue.</a></p>
+</li>
+<li>
+<p><a href="https://issues.apache.org/jira/secure/attachment/12693273/Procedurev2Notification-Bus.pdf">A good doc by Matteo</a> on problem and how Pv2 addresses it w/ roadmap (from the Pv2 JIRA). We should go back to the roadmap to do the Notification Bus, convertion of log splitting to Pv2, etc.</p>
+</li>
+</ul>
+</div>
+</div>
+</div>
+<h1 id="amv2" class="sect0"><a class="anchor" href="#amv2"></a>AMv2 Description for Devs</h1>
+<div class="openblock partintro">
+<div class="content">
+<div class="paragraph">
+<p>The AssignmentManager (AM) in HBase Master manages assignment of Regions over a cluster of RegionServers.</p>
+</div>
+<div class="paragraph">
+<p>The AMv2 project is a redo of Assignment in an attempt at addressing the root cause of many of our operational issues in production, namely slow assignment and problematic accounting such that Regions are misplaced stuck offline in the notorious <em>Regions-In-Transition (RIT)</em> limbo state.</p>
+</div>
+<div class="paragraph">
+<p>Below are notes for devs on key aspects of AMv2 in no particular order.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_background"><a class="anchor" href="#_background"></a>184. Background</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>Assignment in HBase 1.x has been problematic in operation. It is not hard to see why. Region state is kept at the other end of an RPC in ZooKeeper (Terminal states — i.e. OPEN or CLOSED — are published to the <em>hbase:meta</em> table). In HBase-1.x.x, state has multiple writers with Master and RegionServers all able to make state edits concurrently (in <em>hbase:meta</em> table and out on ZooKeeper). If clocks are awry or watchers missed, state changes can be skipped or overwritten. Locking of HBase Entities — tables, regions — is not comprehensive so a table operation — disable/enable — could clash with a region-level operation; a split or merge. Region state is distributed and hard to reason about and test. Assignment is slow in operation because each assign involves moving remote znodes through transitions. Cluster size tends to top out at a couple of hundred thousand r
egions; beyond this, cluster start/stop takes hours and is prone to corruption.</p>
+</div>
+<div class="paragraph">
+<p>AMv2 (AssignmentManager Version 2) is a refactor (<a href="https://issues.apache.org/jira/browse/HBASE-14350">HBASE-14350</a>) of the hbase-1.x AssignmentManager putting it up on a <a href="https://issues.apache.org/jira/browse/HBASE-12439">ProcedureV2 (HBASE-12439)</a> basis. ProcedureV2 (Pv2)<em>,</em> is an awkwardly named system that allows describing and running multi-step state machines. It is performant and persists all state to a Store which is recoverable post crash. See the companion chapter on <a href="#pv2">Procedure Framework (Pv2): <a href="https://issues.apache.org/jira/browse/HBASE-12439">HBASE-12439</a></a>, to learn more about the ProcedureV2 system.</p>
+</div>
+<div class="paragraph">
+<p>In AMv2, all assignment, crash handling, splits and merges are recast as Procedures(v2). ZooKeeper is purged from the mix. As before, the final assignment state gets published to <em>hbase:meta</em> for non-Master participants to read (all-clients) with intermediate state kept in the local Pv2 WAL-based ‘store’ but only the active Master, a single-writer, evolves state. The Master’s in-memory cluster image is the authority and if disagreement, RegionServers are forced to comply. Pv2 adds shared/exclusive locking of all core HBase Entities — namespace, tables, and regions — to ensure one actor at a time access and to prevent operations contending over resources (move/split, disable/assign, etc.).</p>
+</div>
+<div class="paragraph">
+<p>This redo of AM atop of a purposed, performant state machine with all operations taking on the common Procedure form with a single state writer only moves our AM to a new level of resilience and scale.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_new_system"><a class="anchor" href="#_new_system"></a>185. New System</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>Each Region Assign or Unassign of a Region is now a Procedure. A Move (Region) Procedure is a compound of Procedures; it is the running of an Unassign Procedure followed by an Assign Procedure. The Move Procedure spawns the Assign and Unassign in series and then waits on their completions.</p>
+</div>
+<div class="paragraph">
+<p>And so on. ServerCrashProcedure spawns the WAL splitting tasks and then the reassign of all regions that were hosted on the crashed server as subprocedures.</p>
+</div>
+<div class="paragraph">
+<p>AMv2 Procedures are run by the Master in a ProcedureExecutor instance. All Procedures make use of utility provided by the Pv2 framework.</p>
+</div>
+<div class="paragraph">
+<p>For example, Procedures persist each state transition to the frameworks’ Procedure Store. The default implementation is done as a WAL kept on HDFS. On crash, we reopen the Store and rerun all WALs of Procedure transitions to put the Assignment State Machine back into the attitude it had just before crash. We then continue Procedure execution.</p>
+</div>
+<div class="paragraph">
+<p>In the new system, the Master is the Authority on all things Assign. Previous we were ambiguous; e.g. the RegionServer was in charge of Split operations. Master keeps an in-memory image of Region states and servers. If disagreement, the Master always prevails; at an extreme it will kill the RegionServer that is in disagreement.</p>
+</div>
+<div class="paragraph">
+<p>A new RegionStateStore class takes care of publishing the terminal Region state, whether OPEN or CLOSED, out to the _hbase:meta _table<em>.</em></p>
+</div>
+<div class="paragraph">
+<p>RegionServers now report their run version on Connection. This version is available inside the AM for use running migrating rolling restarts.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_procedures_detail"><a class="anchor" href="#_procedures_detail"></a>186. Procedures Detail</h2>
+<div class="sectionbody">
+<div class="sect2">
+<h3 id="_assign_unassign"><a class="anchor" href="#_assign_unassign"></a>186.1. Assign/Unassign</h3>
+<div class="paragraph">
+<p>Assign and Unassign subclass a common RegionTransitionProcedure. There can only be one RegionTransitionProcedure per region running at a time since the RTP instance takes a lock on the region. The RTP base Procedure has three steps; a store the procedure step (REGION_TRANSITION_QUEUE); a dispatch of the procedure open or close followed by a suspend waiting on the remote regionserver to report successful open or fail (REGION_TRANSITION_DISPATCH) or notification that the server fielding the request crashed; and finally registration of the successful open/close in hbase:meta (REGION_TRANSITION_FINISH).</p>
+</div>
+<div class="paragraph">
+<p>Here is how the assign of a region 56f985a727afe80a184dac75fbf6860c looks in the logs. The assign was provoked by a Server Crash (Process ID 1176 or pid=1176 which when it is the parent of a procedure, it is identified as ppid=1176). The assign is pid=1179, the second region of the two being assigned by this Server Crash.</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">24</span>,<span class="integer">175</span> INFO [ProcExecWrkr-<span class="integer">30</span>] procedure2.ProcedureExecutor: Initialized subprocedures=[{pid=<span class="integer">1178</span>, ppid=<span class="integer">1176</span>, state=RUNNABLE:REGION_TRANSITION_QUEUE; AssignProcedure table=IntegrationTestBigLinkedList, region=bfd57f0b72fd3ca77e9d3c5e3ae48d76, target=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>}, {pid=<span class="integer">1179</span>, ppid=<span class="integer">1176</span>, state=RUNNABLE:REGION_TRANSITION_QUEUE; AssignProcedure table=IntegrationTestBigLinkedList, region=<span class="float">56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c, target=ve0540.ha
lxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>}]</code></pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Next we start the assign by queuing (‘registering’) the Procedure with the framework.</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">24</span>,<span class="integer">241</span> INFO [ProcExecWrkr-<span class="integer">30</span>] assignment.AssignProcedure: Start pid=<span class="integer">1179</span>, ppid=<span class="integer">1176</span>, state=RUNNABLE:REGION_TRANSITION_QUEUE; AssignProcedure table=IntegrationTestBigLinkedList, region=<span class="float">56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c, target=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>; rit=OFFLINE, location=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>; forceNewPlan=<span class="predefined-constant">false</span>, retain=<span class="predefined-constant">false</span></code></pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Track the running of Procedures in logs by tracing their process id — here pid=1179.</p>
+</div>
+<div class="paragraph">
+<p>Next we move to the dispatch phase where we update hbase:meta table setting the region state as OPENING on server ve540. We then dispatch an rpc to ve540 asking it to open the region. Thereafter we suspend the Assign until we get a message back from ve540 on whether it has opened the region successfully (or not).</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">24</span>,<span class="integer">494</span> INFO [ProcExecWrkr-<span class="integer">38</span>] assignment.RegionStateStore: pid=<span class="integer">1179</span> updating hbase:meta row=IntegrationTestBigLinkedList,H<span class="error">\</span>xE3<span class="error">@</span><span class="error">\</span>x8D<span class="error">\</span>x964<span class="error">\</span>x9D<span class="error">\</span>xDF<span class="error">\</span>x8F<span class="error">@</span><span class="integer">9</span><span class="error">\</span>x0F<span class="error">\</span>xC8<span class="error">\</span>xCC<span class="error">\</span>xC2,<span class="float">1495566261066.56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c., regionState=OPENING, regionLocation=ve0540.
halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>
+<span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">24</span>,<span class="integer">498</span> INFO [ProcExecWrkr-<span class="integer">38</span>] assignment.RegionTransitionProcedure: Dispatch pid=<span class="integer">1179</span>, ppid=<span class="integer">1176</span>, state=RUNNABLE:REGION_TRANSITION_DISPATCH; AssignProcedure table=IntegrationTestBigLinkedList, region=<span class="float">56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c, target=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>; rit=OPENING, location=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span></code></pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Below we log the incoming report that the region opened successfully on ve540. The Procedure is woken up (you can tell it the procedure is running by the name of the thread, its a ProcedureExecutor thread, ProcExecWrkr-9). The woken up Procedure updates state in hbase:meta to denote the region as open on ve0540. It then reports finished and exits.</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre class="CodeRay highlight"><code data-lang="java"><span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">26</span>,<span class="integer">643</span> DEBUG [RpcServer.default.FPBQ.Fifo.handler=<span class="integer">46</span>,queue=<span class="integer">1</span>,port=<span class="integer">16000</span>] assignment.RegionTransitionProcedure: Received report OPENED seqId=<span class="integer">11984985</span>, pid=<span class="integer">1179</span>, ppid=<span class="integer">1176</span>, state=RUNNABLE:REGION_TRANSITION_DISPATCH; AssignProcedure table=IntegrationTestBigLinkedList, region=<span class="float">56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c, target=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>; rit=OPENING, location=ve0540.halxg.example.org,<span class="integer">16020</spa
n>,<span class="integer">1495525111232</span> <span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">26</span>,<span class="integer">643</span> INFO [ProcExecWrkr-<span class="integer">9</span>] assignment.RegionStateStore: pid=<span class="integer">1179</span> updating hbase:meta row=IntegrationTestBigLinkedList,H<span class="error">\</span>xE3<span class="error">@</span><span class="error">\</span>x8D<span class="error">\</span>x964<span class="error">\</span>x9D<span class="error">\</span>xDF<span class="error">\</span>x8F<span class="error">@</span><span class="integer">9</span><span class="error">\</span>x0F<span class="error">\</span>xC8<span class="error">\</span>xCC<span class="error">\</
span>xC2,<span class="float">1495566261066.56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c., regionState=OPEN, openSeqNum=<span class="integer">11984985</span>, regionLocation=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span>
+<span class="integer">2017</span>-<span class="octal">05</span>-<span class="integer">23</span> <span class="integer">12</span>:<span class="octal">04</span>:<span class="integer">26</span>,<span class="integer">836</span> INFO [ProcExecWrkr-<span class="integer">9</span>] procedure2.ProcedureExecutor: Finish suprocedure pid=<span class="integer">1179</span>, ppid=<span class="integer">1176</span>, state=SUCCESS; AssignProcedure table=IntegrationTestBigLinkedList, region=<span class="float">56f</span><span class="integer">985</span>a727afe80a184dac75fbf6860c, target=ve0540.halxg.example.org,<span class="integer">16020</span>,<span class="integer">1495525111232</span></code></pre>
+</div>
+</div>
+<div class="paragraph">
+<p>Unassign looks similar given it is based on the base RegionTransitionProcedure. It has the same state transitions and does basically the same steps but with different state name (CLOSING, CLOSED).</p>
+</div>
+<div class="paragraph">
+<p>Most other procedures are subclasses of a Pv2 StateMachine implementation. We have both Table and Region focused StateMachines types.</p>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_ui"><a class="anchor" href="#_ui"></a>187. UI</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>Along the top-bar on the Master, you can now find a ‘Procedures&Locks’ tab which takes you to a page that is ugly but useful. It dumps currently running procedures and framework locks. Look at this when you can’t figure what stuff is stuck; it will at least identify problematic procedures (take the pid and grep the logs…). Look for ROLLEDBACK or pids that have been RUNNING for a long time.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_logging"><a class="anchor" href="#_logging"></a>188. Logging</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>Procedures log their process ids as pid= and their parent ids (ppid=) everywhere. Work has been done so you can grep the pid and see history of a procedure operation.</p>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_implementation_notes"><a class="anchor" href="#_implementation_notes"></a>189. Implementation Notes</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>In this section we note some idiosyncrasies of operation as an attempt at saving you some head-scratching.</p>
+</div>
+<div class="sect2">
+<h3 id="_region_transition_rpc_and_rs_heartbeat_can_arrive_at_same_time_on_master"><a class="anchor" href="#_region_transition_rpc_and_rs_heartbeat_can_arrive_at_same_time_on_master"></a>189.1. Region Transition RPC and RS Heartbeat can arrive at ~same time on Master</h3>
+<div class="paragraph">
+<p>Reporting Region Transition on a RegionServer is now a RPC distinct from RS heartbeating (‘RegionServerServices’ Service). An heartbeat and a status update can arrive at the Master at about the same time. The Master will update its internal state for a Region but this same state is checked when heartbeat processing. We may find the unexpected; i.e. a Region just reported as CLOSED so heartbeat is surprised to find region OPEN on the back of the RS report. In the new system, all slaves must cow to the Masters’ understanding of cluster state; the Master will kill/close any misaligned entities.</p>
+</div>
+<div class="paragraph">
+<p>To address the above, we added a lastUpdate for in-memory Master state. Let a region state have some vintage before we act on it (one second currently).</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_master_as_regionserver_or_as_regionserver_that_just_does_system_tables"><a class="anchor" href="#_master_as_regionserver_or_as_regionserver_that_just_does_system_tables"></a>189.2. Master as RegionServer or as RegionServer that just does system tables</h3>
+<div class="paragraph">
+<p>AMv2 enforces current master branch default of HMaster carrying system tables only; i.e. the Master in an HBase cluster acts also as a RegionServer only it is the exclusive host for tables such as <em>hbase:meta</em>, <em>hbase:namespace</em>, etc., the core system tables. This is causing a couple of test failures as AMv1, though it is not supposed to, allows moving hbase:meta off Master while AMv2 does not.</p>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_new_configs"><a class="anchor" href="#_new_configs"></a>190. New Configs</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>These configs all need doc on when you’d change them.</p>
+</div>
+<div class="sect2">
+<h3 id="_hbase_procedure_remote_dispatcher_threadpool_size"><a class="anchor" href="#_hbase_procedure_remote_dispatcher_threadpool_size"></a>190.1. hbase.procedure.remote.dispatcher.threadpool.size</h3>
+<div class="paragraph">
+<p>Defaults 128</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_hbase_procedure_remote_dispatcher_delay_msec"><a class="anchor" href="#_hbase_procedure_remote_dispatcher_delay_msec"></a>190.2. hbase.procedure.remote.dispatcher.delay.msec</h3>
+<div class="paragraph">
+<p>Default 150ms</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_hbase_procedure_remote_dispatcher_max_queue_size"><a class="anchor" href="#_hbase_procedure_remote_dispatcher_max_queue_size"></a>190.3. hbase.procedure.remote.dispatcher.max.queue.size</h3>
+<div class="paragraph">
+<p>Default 32</p>
+</div>
+</div>
+<div class="sect2">
+<h3 id="_hbase_regionserver_rpc_startup_waittime"><a class="anchor" href="#_hbase_regionserver_rpc_startup_waittime"></a>190.4. hbase.regionserver.rpc.startup.waittime</h3>
+<div class="paragraph">
+<p>Default 60 seconds.</p>
+</div>
+</div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_tools"><a class="anchor" href="#_tools"></a>191. Tools</h2>
+<div class="sectionbody">
+<div class="paragraph">
+<p>HBASE-15592 Print Procedure WAL Content</p>
+</div>
+<div class="paragraph">
+<p>Patch in <a href="https://issues.apache.org/jira/browse/HBASE-18152">HBASE-18152</a> [AMv2] Corrupt Procedure WAL file; procedure data stored out of order <a href="https://issues.apache.org/jira/secure/attachment/12871066/reading_bad_wal.patch">https://issues.apache.org/jira/secure/attachment/12871066/reading_bad_wal.patch</a></p>
+</div>
+<div class="sect2">
+<h3 id="_masterprocedureschedulerperformanceevaluation"><a class="anchor" href="#_masterprocedureschedulerperformanceevaluation"></a>191.1. MasterProcedureSchedulerPerformanceEvaluation</h3>
+<div class="paragraph">
+<p>Tool to test performance of locks and queues in procedure scheduler independently from other framework components. Run this after any substantial changes in proc system. Prints nice output:</p>
+</div>
+<div class="listingblock">
+<div class="content">
+<pre>******************************************
+Time - addBack : 5.0600sec
+Ops/sec - addBack : 1.9M
+Time - poll : 19.4590sec
+Ops/sec - poll : 501.9K
+Num Operations : 10000000
+
+Completed : 10000006
+Yield : 22025876
+
+Num Tables : 5
+Regions per table : 10
+Operations type : both
+Threads : 10
+******************************************
+Raw format for scripts
+
+RESULT [num_ops=10000000, ops_type=both, num_table=5, regions_per_table=10, threads=10, num_yield=22025876, time_addback_ms=5060, time_poll_ms=19459]</pre>
+</div>
+</div>
+</div>
+</div>
+</div>
<h1 id="zookeeper" class="sect0"><a class="anchor" href="#zookeeper"></a>ZooKeeper</h1>
<div class="openblock partintro">
<div class="content">
@@ -34031,7 +34558,7 @@ zookeeper could start dropping sessions if it has to run through a directory of
</div>
</div>
<div class="sect1">
-<h2 id="_using_existing_zookeeper_ensemble"><a class="anchor" href="#_using_existing_zookeeper_ensemble"></a>176. Using existing ZooKeeper ensemble</h2>
+<h2 id="_using_existing_zookeeper_ensemble"><a class="anchor" href="#_using_existing_zookeeper_ensemble"></a>192. Using existing ZooKeeper ensemble</h2>
<div class="sectionbody">
<div class="paragraph">
<p>To point HBase at an existing ZooKeeper cluster, one that is not managed by HBase, set <code>HBASE_MANAGES_ZK</code> in <em>conf/hbase-env.sh</em> to false</p>
@@ -34068,7 +34595,7 @@ Additionally, see the <a href="https://wiki.apache.org/hadoop/ZooKeeper/FAQ#A7">
</div>
</div>
<div class="sect1">
-<h2 id="zk.sasl.auth"><a class="anchor" href="#zk.sasl.auth"></a>177. SASL Authentication with ZooKeeper</h2>
+<h2 id="zk.sasl.auth"><a class="anchor" href="#zk.sasl.auth"></a>193. SASL Authentication with ZooKeeper</h2>
<div class="sectionbody">
<div class="paragraph">
<p>Newer releases of Apache HBase (>= 0.92) will support connecting to a ZooKeeper Quorum that supports SASL authentication (which is available in ZooKeeper versions 3.4.0 or later).</p>
@@ -34078,7 +34605,7 @@ Additionally, see the <a href="https://wiki.apache.org/hadoop/ZooKeeper/FAQ#A7">
ZooKeeper/HBase mutual authentication (<a href="https://issues.apache.org/jira/browse/HBASE-2418">HBASE-2418</a>) is required as part of a complete secure HBase configuration (<a href="https://issues.apache.org/jira/browse/HBASE-3025">HBASE-3025</a>). For simplicity of explication, this section ignores additional configuration required (Secure HDFS and Coprocessor configuration). It’s recommended to begin with an HBase-managed ZooKeeper configuration (as opposed to a standalone ZooKeeper quorum) for ease of learning.</p>
</div>
<div class="sect2">
-<h3 id="_operating_system_prerequisites"><a class="anchor" href="#_operating_system_prerequisites"></a>177.1. Operating System Prerequisites</h3>
+<h3 id="_operating_system_prerequisites"><a class="anchor" href="#_operating_system_prerequisites"></a>193.1. Operating System Prerequisites</h3>
<div class="paragraph">
<p>You need to have a working Kerberos KDC setup.
For each <code>$HOST</code> that will run a ZooKeeper server, you should have a principle <code>zookeeper/$HOST</code>.
@@ -34114,7 +34641,7 @@ The ZooKeeper client and server libraries manage their own ticket refreshment by
</div>
</div>
<div class="sect2">
-<h3 id="_hbase_managed_zookeeper_configuration"><a class="anchor" href="#_hbase_managed_zookeeper_configuration"></a>177.2. HBase-managed ZooKeeper Configuration</h3>
+<h3 id="_hbase_managed_zookeeper_configuration"><a class="anchor" href="#_hbase_managed_zookeeper_configuration"></a>193.2. HBase-managed ZooKeeper Configuration</h3>
<div class="paragraph">
<p>On each node that will run a zookeeper, a master, or a regionserver, create a <a href="http://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html">JAAS</a> configuration file in the conf directory of the node’s <em>HBASE_HOME</em> directory that looks like the following:</p>
</div>
@@ -34206,7 +34733,7 @@ bin/hbase regionserver start</pre>
</div>
</div>
<div class="sect2">
-<h3 id="_external_zookeeper_configuration"><a class="anchor" href="#_external_zookeeper_configuration"></a>177.3. External ZooKeeper Configuration</h3>
+<h3 id="_external_zookeeper_configuration"><a class="anchor" href="#_external_zookeeper_configuration"></a>193.3. External ZooKeeper Configuration</h3>
<div class="paragraph">
<p>Add a JAAS configuration file that looks like:</p>
</div>
@@ -34307,7 +34834,7 @@ bin/hbase regionserver start</pre>
</div>
</div>
<div class="sect2">
-<h3 id="_zookeeper_server_authentication_log_output"><a class="anchor" href="#_zookeeper_server_authentication_log_output"></a>177.4. ZooKeeper Server Authentication Log Output</h3>
+<h3 id="_zookeeper_server_authentication_log_output"><a class="anchor" href="#_zookeeper_server_authentication_log_output"></a>193.4. ZooKeeper Server Authentication Log Output</h3>
<div class="paragraph">
<p>If the configuration above is successful, you should see something similar to the following in your ZooKeeper server logs:</p>
</div>
@@ -34329,7 +34856,7 @@ bin/hbase regionserver start</pre>
</div>
</div>
<div class="sect2">
-<h3 id="_zookeeper_client_authentication_log_output"><a class="anchor" href="#_zookeeper_client_authentication_log_output"></a>177.5. ZooKeeper Client Authentication Log Output</h3>
+<h3 id="_zookeeper_client_authentication_log_output"><a class="anchor" href="#_zookeeper_client_authentication_log_output"></a>193.5. ZooKeeper Client Authentication Log Output</h3>
<div class="paragraph">
<p>On the ZooKeeper client side (HBase master or regionserver), you should see something similar to the following:</p>
</div>
@@ -34350,7 +34877,7 @@ bin/hbase regionserver start</pre>
</div>
</div>
<div class="sect2">
-<h3 id="_configuration_from_scratch"><a class="anchor" href="#_configuration_from_scratch"></a>177.6. Configuration from Scratch</h3>
+<h3 id="_configuration_from_scratch"><a class="anchor" href="#_configuration_from_scratch"></a>193.6. Configuration from Scratch</h3>
<div class="paragraph">
<p>This has been tested on the current standard Amazon Linux AMI.
First setup KDC and principals as described above.
@@ -34376,9 +34903,9 @@ bin/hbase regionserver &</pre>
</div>
</div>
<div class="sect2">
-<h3 id="_future_improvements"><a class="anchor" href="#_future_improvements"></a>177.7. Future improvements</h3>
+<h3 id="_future_improvements"><a class="anchor" href="#_future_improvements"></a>193.7. Future improvements</h3>
<div class="sect3">
-<h4 id="_fix_target_cached_classpath_txt"><a class="anchor" href="#_fix_target_cached_classpath_txt"></a>177.7.1. Fix target/cached_classpath.txt</h4>
+<h4 id="_fix_target_cached_classpath_txt"><a class="anchor" href="#_fix_target_cached_classpath_txt"></a>193.7.1. Fix target/cached_classpath.txt</h4>
<div class="paragraph">
<p>You must override the standard hadoop-core jar file from the <code>target/cached_classpath.txt</code> file with the version containing the HADOOP-7070 fix.
You can use the following script to do this:</p>
@@ -34391,13 +34918,13 @@ mv target/tmp.txt target/cached_classpath.txt</pre>
</div>
</div>
<div class="sect3">
-<h4 id="_set_jaas_configuration_programmatically"><a class="anchor" href="#_set_jaas_configuration_programmatically"></a>177.7.2. Set JAAS configuration programmatically</h4>
+<h4 id="_set_jaas_configuration_programmatically"><a class="anchor" href="#_set_jaas_configuration_programmatically"></a>193.7.2. Set JAAS configuration programmatically</h4>
<div class="paragraph">
<p>This would avoid the need for a separate Hadoop jar that fixes <a href="https://issues.apache.org/jira/browse/HADOOP-7070">HADOOP-7070</a>.</p>
</div>
</div>
<div class="sect3">
-<h4 id="_elimination_of_code_kerberos_removehostfromprincipal_code_and_kerberos_removerealmfromprincipal"><a class="anchor" href="#_elimination_of_code_kerberos_removehostfromprincipal_code_and_kerberos_removerealmfromprincipal"></a>177.7.3. Elimination of <code>kerberos.removeHostFromPrincipal</code> and`kerberos.removeRealmFromPrincipal`</h4>
+<h4 id="_elimination_of_code_kerberos_removehostfromprincipal_code_and_kerberos_removerealmfromprincipal"><a class="anchor" href="#_elimination_of_code_kerberos_removehostfromprincipal_code_and_kerberos_removerealmfromprincipal"></a>193.7.3. Elimination of <code>kerberos.removeHostFromPrincipal</code> and`kerberos.removeRealmFromPrincipal`</h4>
</div>
</div>
@@ -34405,7 +34932,7 @@ mv target/tmp.txt target/cached_classpath.txt</pre>
</div>
<h1 id="community" class="sect0"><a class="anchor" href="#community"></a>Community</h1>
<div class="sect1">
-<h2 id="_decisions"><a class="anchor" href="#_decisions"></a>178. Decisions</h2>
+<h2 id="_decisions"><a class="anchor" href="#_decisions"></a>194. Decisions</h2>
<div class="sectionbody">
<div class="paragraph">
<div class="title">Feature Branches</div>
@@ -34453,10 +34980,10 @@ We also are currently in violation of this basic tenet — repli
</div>
</div>
<div class="sect1">
-<h2 id="community.roles"><a class="anchor" href="#community.roles"></a>179. Community Roles</h2>
+<h2 id="community.roles"><a class="anchor" href="#community.roles"></a>195. Community Roles</h2>
<div class="sectionbody">
<div class="sect2">
-<h3 id="_release_managers"><a class="anchor" href="#_release_managers"></a>179.1. Release Managers</h3>
+<h3 id="_release_managers"><a class="anchor" href="#_release_managers"></a>195.1. Release Managers</h3>
<div class="paragraph">
<p>Each maintained release branch has a release manager, who volunteers to coordinate new features and bug fixes are backported to that release.
The release managers are <a href="https://hbase.apache.org/team-list.html">committers</a>.
@@ -34510,7 +35037,7 @@ End-of-life releases are not included in this list.
</div>
</div>
<div class="sect1">
-<h2 id="hbase.commit.msg.format"><a class="anchor" href="#hbase.commit.msg.format"></a>180. Commit Message format</h2>
+<h2 id="hbase.commit.msg.format"><a class="anchor" href="#hbase.commit.msg.format"></a>196. Commit Message format</h2>
<div class="sectionbody">
<div class="paragraph">
<p>We <a href="http://search-hadoop.com/m/Gwxwl10cFHa1">agreed</a> to the following Git commit message format:</p>
@@ -37717,7 +38244,7 @@ The <code>LocalFileSpanReceiver</code> looks in <em>hbase-site.xml</em> for
</div>
</div>
<div class="sect1">
-<h2 id="tracing.client.modifications"><a class="anchor" href="#tracing.client.modifications"></a>181. Client Modifications</h2>
+<h2 id="tracing.client.modifications"><a class="anchor" href="#tracing.client.modifications"></a>197. Client Modifications</h2>
<div class="sectionbody">
<div class="paragraph">
<p>In order to turn on tracing in your client code, you must initialize the module sending spans to receiver once per client process.</p>
@@ -37775,7 +38302,7 @@ See the HTrace <em>README</em> for more information on Samplers.</p>
</div>
</div>
<div class="sect1">
-<h2 id="tracing.client.shell"><a class="anchor" href="#tracing.client.shell"></a>182. Tracing from HBase Shell</h2>
+<h2 id="tracing.client.shell"><a class="anchor" href="#tracing.client.shell"></a>198. Tracing from HBase Shell</h2>
<div class="sectionbody">
<div class="paragraph">
<p>You can use <code>trace</code> command for tracing requests from HBase Shell. <code>trace 'start'</code> command turns on tracing and <code>trace 'stop'</code> command turns off tracing.</p>
@@ -38085,7 +38612,7 @@ The server will return cellblocks compressed using this same compressor as long
<div id="footer">
<div id="footer-text">
Version 3.0.0-SNAPSHOT<br>
-Last updated 2018-06-15 21:26:14 UTC
+Last updated 2018-06-16 21:25:17 UTC
</div>
</div>
</body>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/bulk-loads.html
----------------------------------------------------------------------
diff --git a/bulk-loads.html b/bulk-loads.html
index 5b5250a..da05d81 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Bulk Loads in Apache HBase (TM)
@@ -306,7 +306,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/checkstyle-aggregate.html
----------------------------------------------------------------------
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 03732d1..301958b 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Checkstyle Results</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -284,7 +284,7 @@
<td>3640</td>
<td>0</td>
<td>0</td>
-<td>16190</td></tr></table></div>
+<td>16189</td></tr></table></div>
<div class="section">
<h2><a name="Files"></a>Files</h2>
<table border="0" class="table table-striped">
@@ -9962,7 +9962,7 @@
<td><a href="#org.apache.hadoop.hbase.wal.WALSplitter.java">org/apache/hadoop/hbase/wal/WALSplitter.java</a></td>
<td>0</td>
<td>0</td>
-<td>42</td></tr>
+<td>41</td></tr>
<tr class="b">
<td><a href="#org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.java">org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java</a></td>
<td>0</td>
@@ -10126,7 +10126,7 @@
<ul>
<li>max: <tt>"100"</tt></li>
<li>ignorePattern: <tt>"^package.*|^import.*|a href|href|http://|https://|ftp://|org.apache.thrift.|com.google.protobuf.|hbase.protobuf.generated"</tt></li></ul></td>
-<td>1584</td>
+<td>1583</td>
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td></tr>
<tr class="b">
<td></td>
@@ -123525,115 +123525,115 @@
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>170</td></tr>
+<td>172</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>sizes</td>
<td>LineLength</td>
<td>Line is longer than 100 characters (found 110).</td>
-<td>214</td></tr>
+<td>216</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>226</td></tr>
+<td>228</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'while' construct must use '{}'s.</td>
-<td>279</td></tr>
+<td>281</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'while' construct must use '{}'s.</td>
-<td>293</td></tr>
+<td>295</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'while' construct must use '{}'s.</td>
-<td>300</td></tr>
+<td>302</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'while' construct must use '{}'s.</td>
-<td>316</td></tr>
+<td>318</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'while' construct must use '{}'s.</td>
-<td>323</td></tr>
+<td>325</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>327</td></tr>
+<td>329</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>332</td></tr>
+<td>334</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'method def' child has incorrect indentation level 5, expected level should be 4.</td>
-<td>392</td></tr>
+<td>394</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'try' has incorrect indentation level 10, expected level should be 8.</td>
-<td>452</td></tr>
+<td>454</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'try' child has incorrect indentation level 12, expected level should be 10.</td>
-<td>453</td></tr>
+<td>455</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'try rcurly' has incorrect indentation level 10, expected level should be 8.</td>
-<td>455</td></tr>
+<td>457</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'catch' child has incorrect indentation level 12, expected level should be 10.</td>
-<td>456</td></tr>
+<td>458</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'catch rcurly' has incorrect indentation level 10, expected level should be 8.</td>
-<td>457</td></tr>
+<td>459</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>470</td></tr>
+<td>472</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>535</td></tr>
+<td>537</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>611</td></tr></table></div>
+<td>613</td></tr></table></div>
<div class="section">
<h3 id="org.apache.hadoop.hbase.wal.TestWALFiltering.java">org/apache/hadoop/hbase/wal/TestWALFiltering.java</h3>
<table border="0" class="table table-striped">
@@ -124317,203 +124317,197 @@
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>461</td></tr>
+<td>462</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>465</td></tr>
+<td>464</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>471</td></tr>
+<td>470</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>517</td></tr>
+<td>518</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>518</td></tr>
+<td>519</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>549</td></tr>
+<td>550</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>550</td></tr>
+<td>551</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>552</td></tr>
+<td>553</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>596</td></tr>
+<td>597</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>600</td></tr>
+<td>601</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>694</td></tr>
+<td>695</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>696</td></tr>
+<td>697</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>697</td></tr>
+<td>698</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'throws' has incorrect indentation level 2, expected level should be 4.</td>
-<td>747</td></tr>
+<td>748</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'dataAvailable' must be private and have accessor methods.</td>
-<td>812</td></tr>
+<td>813</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>823</td></tr>
+<td>824</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>868</td></tr>
+<td>869</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>869</td></tr>
+<td>870</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>1158</td></tr>
+<td>1159</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>1203</td></tr>
+<td>1204</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>1241</td></tr>
+<td>1242</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>misc</td>
<td>ArrayTypeStyle</td>
<td>Array brackets at illegal position.</td>
-<td>1462</td></tr>
+<td>1465</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
-<td>sizes</td>
-<td>LineLength</td>
-<td>Line is longer than 100 characters (found 105).</td>
-<td>1488</td></tr>
-<tr class="a">
-<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
<td>'catch' child has incorrect indentation level 10, expected level should be 8.</td>
-<td>1575</td></tr>
-<tr class="b">
+<td>1580</td></tr>
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'type' must be private and have accessor methods.</td>
-<td>1783</td></tr>
-<tr class="a">
+<td>1788</td></tr>
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'mutation' must be private and have accessor methods.</td>
-<td>1784</td></tr>
-<tr class="b">
+<td>1789</td></tr>
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'nonceGroup' must be private and have accessor methods.</td>
-<td>1785</td></tr>
-<tr class="a">
+<td>1790</td></tr>
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'nonce' must be private and have accessor methods.</td>
-<td>1786</td></tr>
-<tr class="b">
+<td>1791</td></tr>
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>1811</td></tr>
-<tr class="a">
+<td>1816</td></tr>
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>1812</td></tr>
-<tr class="b">
+<td>1817</td></tr>
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>1816</td></tr>
-<tr class="a">
+<td>1821</td></tr>
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>1843</td></tr></table></div>
+<td>1848</td></tr></table></div>
<div class="section">
<h3 id="org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.java">org/apache/hadoop/hbase/zookeeper/ReadOnlyZKClient.java</h3>
<table border="0" class="table table-striped">
-<tr class="b">
+<tr class="a">
<th>Severity</th>
<th>Category</th>
<th>Rule</th>
<th>Message</th>
<th>Line</th></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>indentation</td>
<td>Indentation</td>
@@ -124522,25 +124516,25 @@
<div class="section">
<h3 id="org.apache.hadoop.hbase.zookeeper.ZKConfig.java">org/apache/hadoop/hbase/zookeeper/ZKConfig.java</h3>
<table border="0" class="table table-striped">
-<tr class="b">
+<tr class="a">
<th>Severity</th>
<th>Category</th>
<th>Rule</th>
<th>Message</th>
<th>Line</th></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
<td>118</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
<td>187</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
@@ -124549,13 +124543,13 @@
<div class="section">
<h3 id="org.apache.hadoop.hbase.zookeeper.ZKMetadata.java">org/apache/hadoop/hbase/zookeeper/ZKMetadata.java</h3>
<table border="0" class="table table-striped">
-<tr class="b">
+<tr class="a">
<th>Severity</th>
<th>Category</th>
<th>Rule</th>
<th>Message</th>
<th>Line</th></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>FinalClass</td>
@@ -124564,163 +124558,163 @@
<div class="section">
<h3 id="org.apache.hadoop.hbase.zookeeper.ZNodePaths.java">org/apache/hadoop/hbase/zookeeper/ZNodePaths.java</h3>
<table border="0" class="table table-striped">
-<tr class="b">
+<tr class="a">
<th>Severity</th>
<th>Category</th>
<th>Rule</th>
<th>Message</th>
<th>Line</th></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'baseZNode' must be private and have accessor methods.</td>
<td>46</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'metaZNodePrefix' must be private and have accessor methods.</td>
<td>48</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'metaReplicaZNodes' must be private and have accessor methods.</td>
<td>50</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'rsZNode' must be private and have accessor methods.</td>
<td>52</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'drainingZNode' must be private and have accessor methods.</td>
<td>54</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'masterAddressZNode' must be private and have accessor methods.</td>
<td>56</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'backupMasterAddressesZNode' must be private and have accessor methods.</td>
<td>58</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'clusterStateZNode' must be private and have accessor methods.</td>
<td>60</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>annotation</td>
<td>MissingDeprecated</td>
<td>Must include both @java.lang.Deprecated annotation and @deprecated Javadoc tag with description.</td>
<td>65</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'tableZNode' must be private and have accessor methods.</td>
<td>66</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'clusterIdZNode' must be private and have accessor methods.</td>
<td>68</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'splitLogZNode' must be private and have accessor methods.</td>
<td>70</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'balancerZNode' must be private and have accessor methods.</td>
<td>72</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'regionNormalizerZNode' must be private and have accessor methods.</td>
<td>74</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'switchZNode' must be private and have accessor methods.</td>
<td>76</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'tableLockZNode' must be private and have accessor methods.</td>
<td>78</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'namespaceZNode' must be private and have accessor methods.</td>
<td>80</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'masterMaintZNode' must be private and have accessor methods.</td>
<td>82</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'replicationZNode' must be private and have accessor methods.</td>
<td>85</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'peersZNode' must be private and have accessor methods.</td>
<td>87</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'queuesZNode' must be private and have accessor methods.</td>
<td>89</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>design</td>
<td>VisibilityModifier</td>
<td>Variable 'hfileRefsZNode' must be private and have accessor methods.</td>
<td>91</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
<td>144</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
<td>156</td></tr>
-<tr class="a">
+<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
<td>169</td></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
@@ -124729,13 +124723,13 @@
<div class="section">
<h3 id="org.apache.hadoop.hbase.zookeeper.ZooKeeperHelper.java">org/apache/hadoop/hbase/zookeeper/ZooKeeperHelper.java</h3>
<table border="0" class="table table-striped">
-<tr class="a">
+<tr class="b">
<th>Severity</th>
<th>Category</th>
<th>Rule</th>
<th>Message</th>
<th>Line</th></tr>
-<tr class="b">
+<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>imports</td>
<td>ImportOrder</td>
@@ -124753,7 +124747,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/checkstyle.rss
----------------------------------------------------------------------
diff --git a/checkstyle.rss b/checkstyle.rss
index 008388e..9434310 100644
--- a/checkstyle.rss
+++ b/checkstyle.rss
@@ -26,7 +26,7 @@ under the License.
<copyright>©2007 - 2018 The Apache Software Foundation</copyright>
<item>
<title>File: 3640,
- Errors: 16190,
+ Errors: 16189,
Warnings: 0,
Infos: 0
</title>
@@ -48789,7 +48789,7 @@ under the License.
0
</td>
<td>
- 42
+ 41
</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/coc.html
----------------------------------------------------------------------
diff --git a/coc.html b/coc.html
index 7899244..06bc09a 100644
--- a/coc.html
+++ b/coc.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Code of Conduct Policy
@@ -375,7 +375,7 @@ email to <a class="externalLink" href="mailto:private@hbase.apache.org">the priv
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/dependencies.html
----------------------------------------------------------------------
diff --git a/dependencies.html b/dependencies.html
index 7863874..aacc942 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Dependencies</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -440,7 +440,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/dependency-convergence.html
----------------------------------------------------------------------
diff --git a/dependency-convergence.html b/dependency-convergence.html
index c89e071..8400033 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Reactor Dependency Convergence</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -1105,7 +1105,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/dependency-info.html
----------------------------------------------------------------------
diff --git a/dependency-info.html b/dependency-info.html
index 6fe0cda..ea22763 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Dependency Information</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -313,7 +313,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/dependency-management.html
----------------------------------------------------------------------
diff --git a/dependency-management.html b/dependency-management.html
index 8310e50..56a6a2d 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Dependency Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -969,7 +969,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/constant-values.html
----------------------------------------------------------------------
diff --git a/devapidocs/constant-values.html b/devapidocs/constant-values.html
index 521cb44..3b827c1 100644
--- a/devapidocs/constant-values.html
+++ b/devapidocs/constant-values.html
@@ -3768,21 +3768,21 @@
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#date">date</a></code></td>
-<td class="colLast"><code>"Fri Jun 15 21:36:50 UTC 2018"</code></td>
+<td class="colLast"><code>"Sat Jun 16 21:34:54 UTC 2018"</code></td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.revision">
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#revision">revision</a></code></td>
-<td class="colLast"><code>"0e43abc78a5a55c84a7572125d498a950dabfb57"</code></td>
+<td class="colLast"><code>"ac5bb8155b618194fe9cf1131f0e72c99b7b534c"</code></td>
</tr>
<tr class="altColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.srcChecksum">
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#srcChecksum">srcChecksum</a></code></td>
-<td class="colLast"><code>"2e5366d5b65adc25ba60d4644effc4f6"</code></td>
+<td class="colLast"><code>"72ef87fb3fa82a7c510def4b3609400b"</code></td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.url">
[14/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span> }<a nam
<TRUNCATED>
[06/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterThread.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span> }<a nam
<TRUNCATED>
[19/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/index-all.html
----------------------------------------------------------------------
diff --git a/devapidocs/index-all.html b/devapidocs/index-all.html
index 617ca54..c9237ca 100644
--- a/devapidocs/index-all.html
+++ b/devapidocs/index-all.html
@@ -20958,7 +20958,7 @@
<dd>
<div class="block">If you already have a WALFactory, you should favor the instance method.</div>
</dd>
-<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-">createWAP(byte[], WAL.Entry, Path)</a></span> - Method in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></dt>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-">createWAP(byte[], WAL.Entry)</a></span> - Method in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/http/HttpServer.html#createWebAppContext-java.lang.String-org.apache.hadoop.conf.Configuration-org.apache.hadoop.security.authorize.AccessControlList-java.lang.String-">createWebAppContext(String, Configuration, AccessControlList, String)</a></span> - Static method in class org.apache.hadoop.hbase.http.<a href="org/apache/hadoop/hbase/http/HttpServer.html" title="class in org.apache.hadoop.hbase.http">HttpServer</a></dt>
<dd> </dd>
@@ -24507,7 +24507,7 @@
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/querymatcher/ScanDeleteTracker.html#deleteOffset">deleteOffset</a></span> - Variable in class org.apache.hadoop.hbase.regionserver.querymatcher.<a href="org/apache/hadoop/hbase/regionserver/querymatcher/ScanDeleteTracker.html" title="class in org.apache.hadoop.hbase.regionserver.querymatcher">ScanDeleteTracker</a></dt>
<dd> </dd>
-<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#deleteOneWithFewerEntries-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">deleteOneWithFewerEntries(WALSplitter.WriterAndPath, Path)</a></span> - Method in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></dt>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#deleteOneWithFewerEntries-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">deleteOneWithFewerEntries(FileSystem, WALSplitter.WriterAndPath, Path)</a></span> - Method in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.html#deletePath-org.apache.hadoop.fs.Path-">deletePath(Path)</a></span> - Method in class org.apache.hadoop.hbase.mob.compactions.<a href="org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.html" title="class in org.apache.hadoop.hbase.mob.compactions">PartitionedMobCompactor</a></dt>
<dd>
@@ -46849,7 +46849,7 @@
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/master/assignment/RegionStates.html#getRegionsOfTable-org.apache.hadoop.hbase.TableName-boolean-">getRegionsOfTable(TableName, boolean)</a></span> - Method in class org.apache.hadoop.hbase.master.assignment.<a href="org/apache/hadoop/hbase/master/assignment/RegionStates.html" title="class in org.apache.hadoop.hbase.master.assignment">RegionStates</a></dt>
<dd> </dd>
-<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.html#getRegionSplitEditsPath-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-java.lang.String-">getRegionSplitEditsPath(FileSystem, WAL.Entry, Path, String)</a></span> - Static method in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter</a></dt>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.html#getRegionSplitEditsPath-org.apache.hadoop.hbase.wal.WAL.Entry-java.lang.String-org.apache.hadoop.conf.Configuration-">getRegionSplitEditsPath(WAL.Entry, String, Configuration)</a></span> - Static method in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter</a></dt>
<dd>
<div class="block">Path to a file under RECOVERED_EDITS_DIR directory of the region found in
<code>logEntry</code> named for the sequenceid in the passed
@@ -94476,8 +94476,6 @@ service.</div>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/util/FSTableDescriptors.html#rootdir">rootdir</a></span> - Variable in class org.apache.hadoop.hbase.util.<a href="org/apache/hadoop/hbase/util/FSTableDescriptors.html" title="class in org.apache.hadoop.hbase.util">FSTableDescriptors</a></dt>
<dd> </dd>
-<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.html#rootDir">rootDir</a></span> - Variable in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter</a></dt>
-<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html#rootHead">rootHead</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.store.wal.<a href="org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.WalProcedureMap.html" title="class in org.apache.hadoop.hbase.procedure2.store.wal">ProcedureWALFormatReader.WalProcedureMap</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/io/hfile/CacheStats.html#rootIndexHitCount">rootIndexHitCount</a></span> - Variable in class org.apache.hadoop.hbase.io.hfile.<a href="org/apache/hadoop/hbase/io/hfile/CacheStats.html" title="class in org.apache.hadoop.hbase.io.hfile">CacheStats</a></dt>
@@ -120199,6 +120197,8 @@ the order they are declared.</div>
<dd>
<div class="block">WAL directory, where all WAL files would be placed.</div>
</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/wal/WALSplitter.html#walDir">walDir</a></span> - Variable in class org.apache.hadoop.hbase.wal.<a href="org/apache/hadoop/hbase/wal/WALSplitter.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter</a></dt>
+<dd> </dd>
<dt><a href="org/apache/hadoop/hbase/wal/WALEdit.html" title="class in org.apache.hadoop.hbase.wal"><span class="typeNameLink">WALEdit</span></a> - Class in <a href="org/apache/hadoop/hbase/wal/package-summary.html">org.apache.hadoop.hbase.wal</a></dt>
<dd>
<div class="block">WALEdit: Used in HBase's transaction log (WAL) to represent
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/coordination/class-use/SplitLogWorkerCoordination.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/coordination/class-use/SplitLogWorkerCoordination.html b/devapidocs/org/apache/hadoop/hbase/coordination/class-use/SplitLogWorkerCoordination.html
index 00ceaaa..ef24fc2 100644
--- a/devapidocs/org/apache/hadoop/hbase/coordination/class-use/SplitLogWorkerCoordination.html
+++ b/devapidocs/org/apache/hadoop/hbase/coordination/class-use/SplitLogWorkerCoordination.html
@@ -252,7 +252,7 @@
<tbody>
<tr class="altColor">
<td class="colFirst"><code>static boolean</code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path rootDir,
+<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
@@ -274,7 +274,7 @@
<tr class="altColor">
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#WALSplitter-org.apache.hadoop.hbase.wal.WALFactory-org.apache.hadoop.conf.Configuration-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-">WALSplitter</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/wal/WALFactory.html" title="class in org.apache.hadoop.hbase.wal">WALFactory</a> factory,
org.apache.hadoop.conf.Configuration conf,
- org.apache.hadoop.fs.Path rootDir,
+ org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileSystem fs,
<a href="../../../../../../org/apache/hadoop/hbase/regionserver/LastSequenceId.html" title="interface in org.apache.hadoop.hbase.regionserver">LastSequenceId</a> idChecker,
<a href="../../../../../../org/apache/hadoop/hbase/coordination/SplitLogWorkerCoordination.html" title="interface in org.apache.hadoop.hbase.coordination">SplitLogWorkerCoordination</a> splitLogWorkerCoordination)</code> </td>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/LastSequenceId.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/LastSequenceId.html b/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/LastSequenceId.html
index fd91532..d2378b6 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/LastSequenceId.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/class-use/LastSequenceId.html
@@ -184,7 +184,7 @@
<tbody>
<tr class="altColor">
<td class="colFirst"><code>static boolean</code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path rootDir,
+<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
@@ -206,7 +206,7 @@
<tr class="altColor">
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#WALSplitter-org.apache.hadoop.hbase.wal.WALFactory-org.apache.hadoop.conf.Configuration-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-">WALSplitter</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/wal/WALFactory.html" title="class in org.apache.hadoop.hbase.wal">WALFactory</a> factory,
org.apache.hadoop.conf.Configuration conf,
- org.apache.hadoop.fs.Path rootDir,
+ org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileSystem fs,
<a href="../../../../../../org/apache/hadoop/hbase/regionserver/LastSequenceId.html" title="interface in org.apache.hadoop.hbase.regionserver">LastSequenceId</a> idChecker,
<a href="../../../../../../org/apache/hadoop/hbase/coordination/SplitLogWorkerCoordination.html" title="interface in org.apache.hadoop.hbase.coordination">SplitLogWorkerCoordination</a> splitLogWorkerCoordination)</code> </td>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/util/class-use/CancelableProgressable.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/util/class-use/CancelableProgressable.html b/devapidocs/org/apache/hadoop/hbase/util/class-use/CancelableProgressable.html
index ad990ec..5ce2a22 100644
--- a/devapidocs/org/apache/hadoop/hbase/util/class-use/CancelableProgressable.html
+++ b/devapidocs/org/apache/hadoop/hbase/util/class-use/CancelableProgressable.html
@@ -436,7 +436,7 @@
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static boolean</code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path rootDir,
+<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/FSHLogProvider.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/FSHLogProvider.html b/devapidocs/org/apache/hadoop/hbase/wal/FSHLogProvider.html
index 9ff7c1f..ab71c6d 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/FSHLogProvider.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/FSHLogProvider.html
@@ -357,7 +357,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/AbstractFSWALProvide
<ul class="blockList">
<li class="blockList">
<h4>createWAL</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/regionserver/wal/FSHLog.html" title="class in org.apache.hadoop.hbase.regionserver.wal">FSHLog</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html#line.100">createWAL</a>()
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/regionserver/wal/FSHLog.html" title="class in org.apache.hadoop.hbase.regionserver.wal">FSHLog</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html#line.101">createWAL</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -373,7 +373,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/AbstractFSWALProvide
<ul class="blockListLast">
<li class="blockList">
<h4>doInit</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html#line.108">doInit</a>(org.apache.hadoop.conf.Configuration conf)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html#line.109">doInit</a>(org.apache.hadoop.conf.Configuration conf)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
index 0718160..7d373c6 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
@@ -123,7 +123,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1615">WALSplitter.BoundedLogWriterCreationOutputSink</a>
+<pre>class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1620">WALSplitter.BoundedLogWriterCreationOutputSink</a>
extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></pre>
</li>
</ul>
@@ -225,7 +225,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<!-- -->
</a>
<h3>Methods inherited from class org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></h3>
-<code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#appendBuffer-org.apache.hadoop.hbase.wal.WALSplitter.RegionEntryBuffer-boolean-">appendBuffer</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#close--">close</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#closeWriter-java.lang.String-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-java.util.List-">closeWriter</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-">createWAP</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#filterCellByStore-org.apache.hadoop.hbase.wal.WAL.Entry-">filterCellByStore</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink
.html#getWriterAndPath-org.apache.hadoop.hbase.wal.WAL.Entry-boolean-">getWriterAndPath</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#keepRegionEvent-org.apache.hadoop.hbase.wal.WAL.Entry-">keepRegionEvent</a></code></li>
+<code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#appendBuffer-org.apache.hadoop.hbase.wal.WALSplitter.RegionEntryBuffer-boolean-">appendBuffer</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#close--">close</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#closeWriter-java.lang.String-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-java.util.List-">closeWriter</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-">createWAP</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#filterCellByStore-org.apache.hadoop.hbase.wal.WAL.Entry-">filterCellByStore</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#getWriterAndPath-org
.apache.hadoop.hbase.wal.WAL.Entry-boolean-">getWriterAndPath</a>, <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#keepRegionEvent-org.apache.hadoop.hbase.wal.WAL.Entry-">keepRegionEvent</a></code></li>
</ul>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.org.apache.hadoop.hbase.wal.WALSplitter.OutputSink">
@@ -261,7 +261,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockListLast">
<li class="blockList">
<h4>regionRecoverStatMap</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1617">regionRecoverStatMap</a></pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1622">regionRecoverStatMap</a></pre>
</li>
</ul>
</li>
@@ -278,7 +278,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockListLast">
<li class="blockList">
<h4>BoundedLogWriterCreationOutputSink</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1619">BoundedLogWriterCreationOutputSink</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1624">BoundedLogWriterCreationOutputSink</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> entryBuffers,
int numWriters)</pre>
</li>
@@ -297,7 +297,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockList">
<li class="blockList">
<h4>finishWritingAndClose</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1625">finishWritingAndClose</a>()
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1630">finishWritingAndClose</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
@@ -315,7 +315,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockList">
<li class="blockList">
<h4>executeCloseTask</h4>
-<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1640">executeCloseTask</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CompletionService.html?is-external=true" title="class or interface in java.util.concurrent">CompletionService</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>> completionService,
+<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1645">executeCloseTask</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CompletionService.html?is-external=true" title="class or interface in java.util.concurrent">CompletionService</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>> completionService,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> thrown,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> paths)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a>,
@@ -341,7 +341,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockList">
<li class="blockList">
<h4>getOutputCounts</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1673">getOutputCounts</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1678">getOutputCounts</a>()</pre>
<div class="block">since the splitting process may create multiple output files, we need a map
regionRecoverStatMap to track the output count of each region.</div>
<dl>
@@ -358,7 +358,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockList">
<li class="blockList">
<h4>getNumberOfRecoveredRegions</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1685">getNumberOfRecoveredRegions</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1690">getNumberOfRecoveredRegions</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#getNumberOfRecoveredRegions--">getNumberOfRecoveredRegions</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.LogRecoveredEditsOutputSink</a></code></dd>
@@ -373,7 +373,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockList">
<li class="blockList">
<h4>append</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1695">append</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1700">append</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Append the buffer to a new recovered edits file, then close it after all done</div>
<dl>
@@ -392,7 +392,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecov
<ul class="blockListLast">
<li class="blockList">
<h4>writeThenClose</h4>
-<pre>private org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1699">writeThenClose</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
+<pre>private org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html#line.1704">writeThenClose</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
index e4b0655..dd2834a 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
@@ -121,7 +121,7 @@
</dl>
<hr>
<br>
-<pre>static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1762">WALSplitter.CorruptedLogFileException</a>
+<pre>static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1767">WALSplitter.CorruptedLogFileException</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
</li>
</ul>
@@ -205,7 +205,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.h
<ul class="blockListLast">
<li class="blockList">
<h4>serialVersionUID</h4>
-<pre>private static final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html#line.1763">serialVersionUID</a></pre>
+<pre>private static final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html#line.1768">serialVersionUID</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.wal.WALSplitter.CorruptedLogFileException.serialVersionUID">Constant Field Values</a></dd>
@@ -226,7 +226,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.h
<ul class="blockListLast">
<li class="blockList">
<h4>CorruptedLogFileException</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html#line.1765">CorruptedLogFileException</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> s)</pre>
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html#line.1770">CorruptedLogFileException</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> s)</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
index 43be05c..1ac987f 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html
@@ -113,7 +113,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.839">WALSplitter.EntryBuffers</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.840">WALSplitter.EntryBuffers</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">Class which accumulates edits and separates them into a buffer per region
while simultaneously accounting RAM usage. Blocks if the RAM usage crosses
@@ -250,7 +250,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>controller</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.840">controller</a></pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.841">controller</a></pre>
</li>
</ul>
<a name="buffers">
@@ -259,7 +259,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>buffers</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.842">buffers</a></pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.843">buffers</a></pre>
</li>
</ul>
<a name="currentlyWriting">
@@ -268,7 +268,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>currentlyWriting</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><byte[]> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.847">currentlyWriting</a></pre>
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><byte[]> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.848">currentlyWriting</a></pre>
</li>
</ul>
<a name="totalBuffered">
@@ -277,7 +277,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>totalBuffered</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.849">totalBuffered</a></pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.850">totalBuffered</a></pre>
</li>
</ul>
<a name="maxHeapUsage">
@@ -286,7 +286,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>maxHeapUsage</h4>
-<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.850">maxHeapUsage</a></pre>
+<pre>long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.851">maxHeapUsage</a></pre>
</li>
</ul>
<a name="splitWriterCreationBounded">
@@ -295,7 +295,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>splitWriterCreationBounded</h4>
-<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.851">splitWriterCreationBounded</a></pre>
+<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.852">splitWriterCreationBounded</a></pre>
</li>
</ul>
</li>
@@ -312,7 +312,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>EntryBuffers</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.853">EntryBuffers</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.854">EntryBuffers</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
long maxHeapUsage)</pre>
</li>
</ul>
@@ -322,7 +322,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>EntryBuffers</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.857">EntryBuffers</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.858">EntryBuffers</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
long maxHeapUsage,
boolean splitWriterCreationBounded)</pre>
</li>
@@ -341,7 +341,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>appendEntry</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.871">appendEntry</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.872">appendEntry</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a>,
<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Append a log entry into the corresponding region buffer.
@@ -359,7 +359,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getChunkToWrite</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.900">getChunkToWrite</a>()</pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.901">getChunkToWrite</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>RegionEntryBuffer a buffer of edits to be written.</dd>
@@ -372,7 +372,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>doneWriting</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.927">doneWriting</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.928">doneWriting</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)</pre>
</li>
</ul>
<a name="isRegionCurrentlyWriting-byte:A-">
@@ -381,7 +381,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isRegionCurrentlyWriting</h4>
-<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.941">isRegionCurrentlyWriting</a>(byte[] region)</pre>
+<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.942">isRegionCurrentlyWriting</a>(byte[] region)</pre>
</li>
</ul>
<a name="waitUntilDrained--">
@@ -390,7 +390,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>waitUntilDrained</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.945">waitUntilDrained</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html#line.946">waitUntilDrained</a>()</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
index a60261f..5e7fd48 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1227">WALSplitter.LogRecoveredEditsOutputSink</a>
+<pre>class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1228">WALSplitter.LogRecoveredEditsOutputSink</a>
extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.OutputSink</a></pre>
<div class="block">Class that manages the output streams from the log splitting process.</div>
</li>
@@ -204,13 +204,13 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code>(package private) <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a></code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-">createWAP</a></span>(byte[] region,
- <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry,
- org.apache.hadoop.fs.Path rootdir)</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-">createWAP</a></span>(byte[] region,
+ <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</code> </td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>private void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#deleteOneWithFewerEntries-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">deleteOneWithFewerEntries</a></span>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#deleteOneWithFewerEntries-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">deleteOneWithFewerEntries</a></span>(org.apache.hadoop.fs.FileSystem rootFs,
+ <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
org.apache.hadoop.fs.Path dst)</code> </td>
</tr>
<tr id="i7" class="rowColor">
@@ -283,7 +283,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockListLast">
<li class="blockList">
<h4>LogRecoveredEditsOutputSink</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1229">LogRecoveredEditsOutputSink</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1230">LogRecoveredEditsOutputSink</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.PipelineController</a> controller,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.EntryBuffers.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.EntryBuffers</a> entryBuffers,
int numWriters)</pre>
</li>
@@ -302,7 +302,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>finishWritingAndClose</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1244">finishWritingAndClose</a>()
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1245">finishWritingAndClose</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -314,13 +314,14 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
</dl>
</li>
</ul>
-<a name="deleteOneWithFewerEntries-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">
+<a name="deleteOneWithFewerEntries-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>deleteOneWithFewerEntries</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1263">deleteOneWithFewerEntries</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1264">deleteOneWithFewerEntries</a>(org.apache.hadoop.fs.FileSystem rootFs,
+ <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
org.apache.hadoop.fs.Path dst)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -335,7 +336,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>close</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1296">close</a>()
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1298">close</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Close all of the output streams.</div>
<dl>
@@ -352,7 +353,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>executeCloseTask</h4>
-<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1340">executeCloseTask</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CompletionService.html?is-external=true" title="class or interface in java.util.concurrent">CompletionService</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>> completionService,
+<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1342">executeCloseTask</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CompletionService.html?is-external=true" title="class or interface in java.util.concurrent">CompletionService</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>> completionService,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> thrown,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> paths)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a>,
@@ -376,7 +377,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>closeWriter</h4>
-<pre>org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1367">closeWriter</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> encodedRegionName,
+<pre>org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1369">closeWriter</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> encodedRegionName,
<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> thrown)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -392,7 +393,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>closeLogWriters</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1414">closeLogWriters</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> thrown)
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1417">closeLogWriters</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>> thrown)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -406,7 +407,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>getWriterAndPath</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1461">getWriterAndPath</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry,
+<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1464">getWriterAndPath</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry,
boolean reusable)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Get a writer and path for a log starting at the given entry. This function is threadsafe so
@@ -419,15 +420,14 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
</dl>
</li>
</ul>
-<a name="createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-">
+<a name="createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>createWAP</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1487">createWAP</a>(byte[] region,
- <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry,
- org.apache.hadoop.fs.Path rootdir)
+<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1490">createWAP</a>(byte[] region,
+ <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -443,7 +443,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>filterCellByStore</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1505">filterCellByStore</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1510">filterCellByStore</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry)</pre>
</li>
</ul>
<a name="append-org.apache.hadoop.hbase.wal.WALSplitter.RegionEntryBuffer-">
@@ -452,7 +452,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>append</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1535">append</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1540">append</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -470,7 +470,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>appendBuffer</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1539">appendBuffer</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer,
+<pre><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1544">appendBuffer</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.RegionEntryBuffer</a> buffer,
boolean reusable)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -485,7 +485,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>keepRegionEvent</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1584">keepRegionEvent</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1589">keepRegionEvent</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#keepRegionEvent-org.apache.hadoop.hbase.wal.WAL.Entry-">WALSplitter.OutputSink</a></code></span></div>
<div class="block">Some WALEdit's contain only KV's for account on what happened to a region.
Not all sinks will want to get all of those edits.</div>
@@ -503,7 +503,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockList">
<li class="blockList">
<h4>getOutputCounts</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1598">getOutputCounts</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><byte[],<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1603">getOutputCounts</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#getOutputCounts--">getOutputCounts</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.OutputSink</a></code></dd>
@@ -518,7 +518,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSi
<ul class="blockListLast">
<li class="blockList">
<h4>getNumberOfRecoveredRegions</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1607">getNumberOfRecoveredRegions</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#line.1612">getNumberOfRecoveredRegions</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html#getNumberOfRecoveredRegions--">getNumberOfRecoveredRegions</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.OutputSink.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.OutputSink</a></code></dd>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
index 4c42cdc..868d355 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1771">WALSplitter.MutationReplay</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1776">WALSplitter.MutationReplay</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.MutationReplay</a>></pre>
<div class="block">A struct used by getMutationsFromWALEntry</div>
@@ -230,7 +230,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockList">
<li class="blockList">
<h4>type</h4>
-<pre>public final org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1783">type</a></pre>
+<pre>public final org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1788">type</a></pre>
</li>
</ul>
<a name="mutation">
@@ -239,7 +239,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockList">
<li class="blockList">
<h4>mutation</h4>
-<pre>public final <a href="../../../../../org/apache/hadoop/hbase/client/Mutation.html" title="class in org.apache.hadoop.hbase.client">Mutation</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1784">mutation</a></pre>
+<pre>public final <a href="../../../../../org/apache/hadoop/hbase/client/Mutation.html" title="class in org.apache.hadoop.hbase.client">Mutation</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1789">mutation</a></pre>
</li>
</ul>
<a name="nonceGroup">
@@ -248,7 +248,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockList">
<li class="blockList">
<h4>nonceGroup</h4>
-<pre>public final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1785">nonceGroup</a></pre>
+<pre>public final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1790">nonceGroup</a></pre>
</li>
</ul>
<a name="nonce">
@@ -257,7 +257,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockListLast">
<li class="blockList">
<h4>nonce</h4>
-<pre>public final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1786">nonce</a></pre>
+<pre>public final long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1791">nonce</a></pre>
</li>
</ul>
</li>
@@ -274,7 +274,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockListLast">
<li class="blockList">
<h4>MutationReplay</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1772">MutationReplay</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType type,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1777">MutationReplay</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType type,
<a href="../../../../../org/apache/hadoop/hbase/client/Mutation.html" title="class in org.apache.hadoop.hbase.client">Mutation</a> mutation,
long nonceGroup,
long nonce)</pre>
@@ -294,7 +294,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockList">
<li class="blockList">
<h4>compareTo</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1789">compareTo</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.MutationReplay</a> d)</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1794">compareTo</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.MutationReplay</a> d)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true#compareTo-T-" title="class or interface in java.lang">compareTo</a></code> in interface <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.MutationReplay</a>></code></dd>
@@ -307,7 +307,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockList">
<li class="blockList">
<h4>equals</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1794">equals</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a> obj)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1799">equals</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a> obj)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#equals-java.lang.Object-" title="class or interface in java.lang">equals</a></code> in class <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></code></dd>
@@ -320,7 +320,7 @@ implements <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparab
<ul class="blockListLast">
<li class="blockList">
<h4>hashCode</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1803">hashCode</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html#line.1808">hashCode</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true#hashCode--" title="class or interface in java.lang">hashCode</a></code> in class <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></code></dd>
[15/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="l
<TRUNCATED>
[02/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html
index db8cddf..345e682 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html
@@ -65,621 +65,623 @@
<span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.testclassification.MediumTests;<a name="line.57"></a>
<span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.testclassification.RegionServerTests;<a name="line.58"></a>
<span class="sourceLineNo">059</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.59"></a>
-<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.60"></a>
-<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.61"></a>
-<span class="sourceLineNo">062</span>import org.apache.hadoop.hdfs.DistributedFileSystem;<a name="line.62"></a>
-<span class="sourceLineNo">063</span>import org.apache.hadoop.hdfs.MiniDFSCluster;<a name="line.63"></a>
-<span class="sourceLineNo">064</span>import org.apache.hadoop.hdfs.protocol.HdfsConstants;<a name="line.64"></a>
-<span class="sourceLineNo">065</span>import org.junit.After;<a name="line.65"></a>
-<span class="sourceLineNo">066</span>import org.junit.AfterClass;<a name="line.66"></a>
-<span class="sourceLineNo">067</span>import org.junit.Before;<a name="line.67"></a>
-<span class="sourceLineNo">068</span>import org.junit.BeforeClass;<a name="line.68"></a>
-<span class="sourceLineNo">069</span>import org.junit.ClassRule;<a name="line.69"></a>
-<span class="sourceLineNo">070</span>import org.junit.Rule;<a name="line.70"></a>
-<span class="sourceLineNo">071</span>import org.junit.Test;<a name="line.71"></a>
-<span class="sourceLineNo">072</span>import org.junit.experimental.categories.Category;<a name="line.72"></a>
-<span class="sourceLineNo">073</span>import org.junit.rules.TestName;<a name="line.73"></a>
-<span class="sourceLineNo">074</span>import org.slf4j.Logger;<a name="line.74"></a>
-<span class="sourceLineNo">075</span>import org.slf4j.LoggerFactory;<a name="line.75"></a>
-<span class="sourceLineNo">076</span><a name="line.76"></a>
-<span class="sourceLineNo">077</span>/**<a name="line.77"></a>
-<span class="sourceLineNo">078</span> * WAL tests that can be reused across providers.<a name="line.78"></a>
-<span class="sourceLineNo">079</span> */<a name="line.79"></a>
-<span class="sourceLineNo">080</span>@Category({RegionServerTests.class, MediumTests.class})<a name="line.80"></a>
-<span class="sourceLineNo">081</span>public class TestWALFactory {<a name="line.81"></a>
-<span class="sourceLineNo">082</span><a name="line.82"></a>
-<span class="sourceLineNo">083</span> @ClassRule<a name="line.83"></a>
-<span class="sourceLineNo">084</span> public static final HBaseClassTestRule CLASS_RULE =<a name="line.84"></a>
-<span class="sourceLineNo">085</span> HBaseClassTestRule.forClass(TestWALFactory.class);<a name="line.85"></a>
-<span class="sourceLineNo">086</span><a name="line.86"></a>
-<span class="sourceLineNo">087</span> private static final Logger LOG = LoggerFactory.getLogger(TestWALFactory.class);<a name="line.87"></a>
-<span class="sourceLineNo">088</span><a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected static Configuration conf;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> private static MiniDFSCluster cluster;<a name="line.90"></a>
-<span class="sourceLineNo">091</span> protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();<a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected static Path hbaseDir;<a name="line.92"></a>
-<span class="sourceLineNo">093</span> protected static Path hbaseWALDir;<a name="line.93"></a>
-<span class="sourceLineNo">094</span><a name="line.94"></a>
-<span class="sourceLineNo">095</span> protected FileSystem fs;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> protected Path dir;<a name="line.96"></a>
-<span class="sourceLineNo">097</span> protected WALFactory wals;<a name="line.97"></a>
-<span class="sourceLineNo">098</span> private ServerName currentServername;<a name="line.98"></a>
-<span class="sourceLineNo">099</span><a name="line.99"></a>
-<span class="sourceLineNo">100</span> @Rule<a name="line.100"></a>
-<span class="sourceLineNo">101</span> public final TestName currentTest = new TestName();<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span> @Before<a name="line.103"></a>
-<span class="sourceLineNo">104</span> public void setUp() throws Exception {<a name="line.104"></a>
-<span class="sourceLineNo">105</span> fs = cluster.getFileSystem();<a name="line.105"></a>
-<span class="sourceLineNo">106</span> dir = new Path(hbaseDir, currentTest.getMethodName());<a name="line.106"></a>
-<span class="sourceLineNo">107</span> this.currentServername = ServerName.valueOf(currentTest.getMethodName(), 16010, 1);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> wals = new WALFactory(conf, this.currentServername.toString());<a name="line.108"></a>
-<span class="sourceLineNo">109</span> }<a name="line.109"></a>
-<span class="sourceLineNo">110</span><a name="line.110"></a>
-<span class="sourceLineNo">111</span> @After<a name="line.111"></a>
-<span class="sourceLineNo">112</span> public void tearDown() throws Exception {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> // testAppendClose closes the FileSystem, which will prevent us from closing cleanly here.<a name="line.113"></a>
-<span class="sourceLineNo">114</span> try {<a name="line.114"></a>
-<span class="sourceLineNo">115</span> wals.close();<a name="line.115"></a>
-<span class="sourceLineNo">116</span> } catch (IOException exception) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> LOG.warn("Encountered exception while closing wal factory. If you have other errors, this" +<a name="line.117"></a>
-<span class="sourceLineNo">118</span> " may be the cause. Message: " + exception);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> LOG.debug("Exception details for failure to close wal factory.", exception);<a name="line.119"></a>
-<span class="sourceLineNo">120</span> }<a name="line.120"></a>
-<span class="sourceLineNo">121</span> FileStatus[] entries = fs.listStatus(new Path("/"));<a name="line.121"></a>
-<span class="sourceLineNo">122</span> for (FileStatus dir : entries) {<a name="line.122"></a>
-<span class="sourceLineNo">123</span> fs.delete(dir.getPath(), true);<a name="line.123"></a>
-<span class="sourceLineNo">124</span> }<a name="line.124"></a>
-<span class="sourceLineNo">125</span> }<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span> @BeforeClass<a name="line.127"></a>
-<span class="sourceLineNo">128</span> public static void setUpBeforeClass() throws Exception {<a name="line.128"></a>
-<span class="sourceLineNo">129</span> // Make block sizes small.<a name="line.129"></a>
-<span class="sourceLineNo">130</span> TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);<a name="line.130"></a>
-<span class="sourceLineNo">131</span> // needed for testAppendClose()<a name="line.131"></a>
-<span class="sourceLineNo">132</span> // quicker heartbeat interval for faster DN death notification<a name="line.132"></a>
-<span class="sourceLineNo">133</span> TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);<a name="line.133"></a>
-<span class="sourceLineNo">134</span> TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);<a name="line.134"></a>
-<span class="sourceLineNo">135</span> TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> // faster failover with cluster.shutdown();fs.close() idiom<a name="line.137"></a>
-<span class="sourceLineNo">138</span> TEST_UTIL.getConfiguration()<a name="line.138"></a>
-<span class="sourceLineNo">139</span> .setInt("hbase.ipc.client.connect.max.retries", 1);<a name="line.139"></a>
-<span class="sourceLineNo">140</span> TEST_UTIL.getConfiguration().setInt(<a name="line.140"></a>
-<span class="sourceLineNo">141</span> "dfs.client.block.recovery.retries", 1);<a name="line.141"></a>
+<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.util.CommonFSUtils;<a name="line.60"></a>
+<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.61"></a>
+<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.62"></a>
+<span class="sourceLineNo">063</span>import org.apache.hadoop.hdfs.DistributedFileSystem;<a name="line.63"></a>
+<span class="sourceLineNo">064</span>import org.apache.hadoop.hdfs.MiniDFSCluster;<a name="line.64"></a>
+<span class="sourceLineNo">065</span>import org.apache.hadoop.hdfs.protocol.HdfsConstants;<a name="line.65"></a>
+<span class="sourceLineNo">066</span>import org.junit.After;<a name="line.66"></a>
+<span class="sourceLineNo">067</span>import org.junit.AfterClass;<a name="line.67"></a>
+<span class="sourceLineNo">068</span>import org.junit.Before;<a name="line.68"></a>
+<span class="sourceLineNo">069</span>import org.junit.BeforeClass;<a name="line.69"></a>
+<span class="sourceLineNo">070</span>import org.junit.ClassRule;<a name="line.70"></a>
+<span class="sourceLineNo">071</span>import org.junit.Rule;<a name="line.71"></a>
+<span class="sourceLineNo">072</span>import org.junit.Test;<a name="line.72"></a>
+<span class="sourceLineNo">073</span>import org.junit.experimental.categories.Category;<a name="line.73"></a>
+<span class="sourceLineNo">074</span>import org.junit.rules.TestName;<a name="line.74"></a>
+<span class="sourceLineNo">075</span>import org.slf4j.Logger;<a name="line.75"></a>
+<span class="sourceLineNo">076</span>import org.slf4j.LoggerFactory;<a name="line.76"></a>
+<span class="sourceLineNo">077</span><a name="line.77"></a>
+<span class="sourceLineNo">078</span>/**<a name="line.78"></a>
+<span class="sourceLineNo">079</span> * WAL tests that can be reused across providers.<a name="line.79"></a>
+<span class="sourceLineNo">080</span> */<a name="line.80"></a>
+<span class="sourceLineNo">081</span>@Category({RegionServerTests.class, MediumTests.class})<a name="line.81"></a>
+<span class="sourceLineNo">082</span>public class TestWALFactory {<a name="line.82"></a>
+<span class="sourceLineNo">083</span><a name="line.83"></a>
+<span class="sourceLineNo">084</span> @ClassRule<a name="line.84"></a>
+<span class="sourceLineNo">085</span> public static final HBaseClassTestRule CLASS_RULE =<a name="line.85"></a>
+<span class="sourceLineNo">086</span> HBaseClassTestRule.forClass(TestWALFactory.class);<a name="line.86"></a>
+<span class="sourceLineNo">087</span><a name="line.87"></a>
+<span class="sourceLineNo">088</span> private static final Logger LOG = LoggerFactory.getLogger(TestWALFactory.class);<a name="line.88"></a>
+<span class="sourceLineNo">089</span><a name="line.89"></a>
+<span class="sourceLineNo">090</span> protected static Configuration conf;<a name="line.90"></a>
+<span class="sourceLineNo">091</span> private static MiniDFSCluster cluster;<a name="line.91"></a>
+<span class="sourceLineNo">092</span> protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();<a name="line.92"></a>
+<span class="sourceLineNo">093</span> protected static Path hbaseDir;<a name="line.93"></a>
+<span class="sourceLineNo">094</span> protected static Path hbaseWALDir;<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span> protected FileSystem fs;<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected Path dir;<a name="line.97"></a>
+<span class="sourceLineNo">098</span> protected WALFactory wals;<a name="line.98"></a>
+<span class="sourceLineNo">099</span> private ServerName currentServername;<a name="line.99"></a>
+<span class="sourceLineNo">100</span><a name="line.100"></a>
+<span class="sourceLineNo">101</span> @Rule<a name="line.101"></a>
+<span class="sourceLineNo">102</span> public final TestName currentTest = new TestName();<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span> @Before<a name="line.104"></a>
+<span class="sourceLineNo">105</span> public void setUp() throws Exception {<a name="line.105"></a>
+<span class="sourceLineNo">106</span> fs = cluster.getFileSystem();<a name="line.106"></a>
+<span class="sourceLineNo">107</span> dir = new Path(hbaseDir, currentTest.getMethodName());<a name="line.107"></a>
+<span class="sourceLineNo">108</span> this.currentServername = ServerName.valueOf(currentTest.getMethodName(), 16010, 1);<a name="line.108"></a>
+<span class="sourceLineNo">109</span> wals = new WALFactory(conf, this.currentServername.toString());<a name="line.109"></a>
+<span class="sourceLineNo">110</span> }<a name="line.110"></a>
+<span class="sourceLineNo">111</span><a name="line.111"></a>
+<span class="sourceLineNo">112</span> @After<a name="line.112"></a>
+<span class="sourceLineNo">113</span> public void tearDown() throws Exception {<a name="line.113"></a>
+<span class="sourceLineNo">114</span> // testAppendClose closes the FileSystem, which will prevent us from closing cleanly here.<a name="line.114"></a>
+<span class="sourceLineNo">115</span> try {<a name="line.115"></a>
+<span class="sourceLineNo">116</span> wals.close();<a name="line.116"></a>
+<span class="sourceLineNo">117</span> } catch (IOException exception) {<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Encountered exception while closing wal factory. If you have other errors, this" +<a name="line.118"></a>
+<span class="sourceLineNo">119</span> " may be the cause. Message: " + exception);<a name="line.119"></a>
+<span class="sourceLineNo">120</span> LOG.debug("Exception details for failure to close wal factory.", exception);<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span> FileStatus[] entries = fs.listStatus(new Path("/"));<a name="line.122"></a>
+<span class="sourceLineNo">123</span> for (FileStatus dir : entries) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> fs.delete(dir.getPath(), true);<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span> }<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span> @BeforeClass<a name="line.128"></a>
+<span class="sourceLineNo">129</span> public static void setUpBeforeClass() throws Exception {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> CommonFSUtils.setWALRootDir(TEST_UTIL.getConfiguration(), new Path("file:///tmp/wal"));<a name="line.130"></a>
+<span class="sourceLineNo">131</span> // Make block sizes small.<a name="line.131"></a>
+<span class="sourceLineNo">132</span> TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);<a name="line.132"></a>
+<span class="sourceLineNo">133</span> // needed for testAppendClose()<a name="line.133"></a>
+<span class="sourceLineNo">134</span> // quicker heartbeat interval for faster DN death notification<a name="line.134"></a>
+<span class="sourceLineNo">135</span> TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);<a name="line.135"></a>
+<span class="sourceLineNo">136</span> TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);<a name="line.136"></a>
+<span class="sourceLineNo">137</span> TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);<a name="line.137"></a>
+<span class="sourceLineNo">138</span><a name="line.138"></a>
+<span class="sourceLineNo">139</span> // faster failover with cluster.shutdown();fs.close() idiom<a name="line.139"></a>
+<span class="sourceLineNo">140</span> TEST_UTIL.getConfiguration()<a name="line.140"></a>
+<span class="sourceLineNo">141</span> .setInt("hbase.ipc.client.connect.max.retries", 1);<a name="line.141"></a>
<span class="sourceLineNo">142</span> TEST_UTIL.getConfiguration().setInt(<a name="line.142"></a>
-<span class="sourceLineNo">143</span> "hbase.ipc.client.connection.maxidletime", 500);<a name="line.143"></a>
-<span class="sourceLineNo">144</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.timeout", 10000);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 1000);<a name="line.145"></a>
-<span class="sourceLineNo">146</span> TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,<a name="line.146"></a>
-<span class="sourceLineNo">147</span> SampleRegionWALCoprocessor.class.getName());<a name="line.147"></a>
-<span class="sourceLineNo">148</span> TEST_UTIL.startMiniDFSCluster(3);<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span> conf = TEST_UTIL.getConfiguration();<a name="line.150"></a>
-<span class="sourceLineNo">151</span> cluster = TEST_UTIL.getDFSCluster();<a name="line.151"></a>
-<span class="sourceLineNo">152</span><a name="line.152"></a>
-<span class="sourceLineNo">153</span> hbaseDir = TEST_UTIL.createRootDir();<a name="line.153"></a>
-<span class="sourceLineNo">154</span> hbaseWALDir = TEST_UTIL.createWALRootDir();<a name="line.154"></a>
-<span class="sourceLineNo">155</span> }<a name="line.155"></a>
-<span class="sourceLineNo">156</span><a name="line.156"></a>
-<span class="sourceLineNo">157</span> @AfterClass<a name="line.157"></a>
-<span class="sourceLineNo">158</span> public static void tearDownAfterClass() throws Exception {<a name="line.158"></a>
-<span class="sourceLineNo">159</span> TEST_UTIL.shutdownMiniCluster();<a name="line.159"></a>
-<span class="sourceLineNo">160</span> }<a name="line.160"></a>
-<span class="sourceLineNo">161</span><a name="line.161"></a>
-<span class="sourceLineNo">162</span> @Test<a name="line.162"></a>
-<span class="sourceLineNo">163</span> public void canCloseSingleton() throws IOException {<a name="line.163"></a>
-<span class="sourceLineNo">164</span> WALFactory.getInstance(conf).close();<a name="line.164"></a>
-<span class="sourceLineNo">165</span> }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> /**<a name="line.167"></a>
-<span class="sourceLineNo">168</span> * Just write multiple logs then split. Before fix for HADOOP-2283, this<a name="line.168"></a>
-<span class="sourceLineNo">169</span> * would fail.<a name="line.169"></a>
-<span class="sourceLineNo">170</span> * @throws IOException<a name="line.170"></a>
-<span class="sourceLineNo">171</span> */<a name="line.171"></a>
-<span class="sourceLineNo">172</span> @Test<a name="line.172"></a>
-<span class="sourceLineNo">173</span> public void testSplit() throws IOException {<a name="line.173"></a>
-<span class="sourceLineNo">174</span> final TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.174"></a>
-<span class="sourceLineNo">175</span> final byte [] rowName = tableName.getName();<a name="line.175"></a>
-<span class="sourceLineNo">176</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.176"></a>
-<span class="sourceLineNo">177</span> final int howmany = 3;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> RegionInfo[] infos = new RegionInfo[3];<a name="line.178"></a>
-<span class="sourceLineNo">179</span> Path tabledir = FSUtils.getTableDir(hbaseWALDir, tableName);<a name="line.179"></a>
-<span class="sourceLineNo">180</span> fs.mkdirs(tabledir);<a name="line.180"></a>
-<span class="sourceLineNo">181</span> for (int i = 0; i < howmany; i++) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> infos[i] = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("" + i))<a name="line.182"></a>
-<span class="sourceLineNo">183</span> .setEndKey(Bytes.toBytes("" + (i + 1))).build();<a name="line.183"></a>
-<span class="sourceLineNo">184</span> fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));<a name="line.184"></a>
-<span class="sourceLineNo">185</span> LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());<a name="line.185"></a>
-<span class="sourceLineNo">186</span> }<a name="line.186"></a>
-<span class="sourceLineNo">187</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.187"></a>
-<span class="sourceLineNo">188</span> scopes.put(Bytes.toBytes("column"), 0);<a name="line.188"></a>
-<span class="sourceLineNo">189</span><a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span> // Add edits for three regions.<a name="line.191"></a>
-<span class="sourceLineNo">192</span> for (int ii = 0; ii < howmany; ii++) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> for (int i = 0; i < howmany; i++) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> final WAL log =<a name="line.194"></a>
-<span class="sourceLineNo">195</span> wals.getWAL(infos[i]);<a name="line.195"></a>
-<span class="sourceLineNo">196</span> for (int j = 0; j < howmany; j++) {<a name="line.196"></a>
-<span class="sourceLineNo">197</span> WALEdit edit = new WALEdit();<a name="line.197"></a>
-<span class="sourceLineNo">198</span> byte [] family = Bytes.toBytes("column");<a name="line.198"></a>
-<span class="sourceLineNo">199</span> byte [] qualifier = Bytes.toBytes(Integer.toString(j));<a name="line.199"></a>
-<span class="sourceLineNo">200</span> byte [] column = Bytes.toBytes("column:" + Integer.toString(j));<a name="line.200"></a>
-<span class="sourceLineNo">201</span> edit.add(new KeyValue(rowName, family, qualifier,<a name="line.201"></a>
-<span class="sourceLineNo">202</span> System.currentTimeMillis(), column));<a name="line.202"></a>
-<span class="sourceLineNo">203</span> LOG.info("Region " + i + ": " + edit);<a name="line.203"></a>
-<span class="sourceLineNo">204</span> WALKeyImpl walKey = new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName,<a name="line.204"></a>
-<span class="sourceLineNo">205</span> System.currentTimeMillis(), mvcc, scopes);<a name="line.205"></a>
-<span class="sourceLineNo">206</span> log.append(infos[i], walKey, edit, true);<a name="line.206"></a>
-<span class="sourceLineNo">207</span> walKey.getWriteEntry();<a name="line.207"></a>
-<span class="sourceLineNo">208</span> }<a name="line.208"></a>
-<span class="sourceLineNo">209</span> log.sync();<a name="line.209"></a>
-<span class="sourceLineNo">210</span> log.rollWriter(true);<a name="line.210"></a>
-<span class="sourceLineNo">211</span> }<a name="line.211"></a>
-<span class="sourceLineNo">212</span> }<a name="line.212"></a>
-<span class="sourceLineNo">213</span> wals.shutdown();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> // The below calculation of logDir relies on insider information... WALSplitter should be connected better<a name="line.214"></a>
-<span class="sourceLineNo">215</span> // with the WAL system.... not requiring explicit path. The oldLogDir is just made up not used.<a name="line.215"></a>
-<span class="sourceLineNo">216</span> Path logDir =<a name="line.216"></a>
-<span class="sourceLineNo">217</span> new Path(new Path(hbaseWALDir, HConstants.HREGION_LOGDIR_NAME),<a name="line.217"></a>
-<span class="sourceLineNo">218</span> this.currentServername.toString());<a name="line.218"></a>
-<span class="sourceLineNo">219</span> Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.219"></a>
-<span class="sourceLineNo">220</span> List<Path> splits = WALSplitter.split(hbaseWALDir, logDir, oldLogDir, fs, conf, wals);<a name="line.220"></a>
-<span class="sourceLineNo">221</span> verifySplits(splits, howmany);<a name="line.221"></a>
-<span class="sourceLineNo">222</span> }<a name="line.222"></a>
-<span class="sourceLineNo">223</span><a name="line.223"></a>
-<span class="sourceLineNo">224</span> /**<a name="line.224"></a>
-<span class="sourceLineNo">225</span> * Test new HDFS-265 sync.<a name="line.225"></a>
-<span class="sourceLineNo">226</span> * @throws Exception<a name="line.226"></a>
-<span class="sourceLineNo">227</span> */<a name="line.227"></a>
-<span class="sourceLineNo">228</span> @Test<a name="line.228"></a>
-<span class="sourceLineNo">229</span> public void Broken_testSync() throws Exception {<a name="line.229"></a>
-<span class="sourceLineNo">230</span> TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.230"></a>
-<span class="sourceLineNo">231</span> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.231"></a>
-<span class="sourceLineNo">232</span> // First verify that using streams all works.<a name="line.232"></a>
-<span class="sourceLineNo">233</span> Path p = new Path(dir, currentTest.getMethodName() + ".fsdos");<a name="line.233"></a>
-<span class="sourceLineNo">234</span> FSDataOutputStream out = fs.create(p);<a name="line.234"></a>
-<span class="sourceLineNo">235</span> out.write(tableName.getName());<a name="line.235"></a>
-<span class="sourceLineNo">236</span> Method syncMethod = null;<a name="line.236"></a>
-<span class="sourceLineNo">237</span> try {<a name="line.237"></a>
-<span class="sourceLineNo">238</span> syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});<a name="line.238"></a>
-<span class="sourceLineNo">239</span> } catch (NoSuchMethodException e) {<a name="line.239"></a>
-<span class="sourceLineNo">240</span> try {<a name="line.240"></a>
-<span class="sourceLineNo">241</span> syncMethod = out.getClass().getMethod("sync", new Class<?> []{});<a name="line.241"></a>
-<span class="sourceLineNo">242</span> } catch (NoSuchMethodException ex) {<a name="line.242"></a>
-<span class="sourceLineNo">243</span> fail("This version of Hadoop supports neither Syncable.sync() " +<a name="line.243"></a>
-<span class="sourceLineNo">244</span> "nor Syncable.hflush().");<a name="line.244"></a>
-<span class="sourceLineNo">245</span> }<a name="line.245"></a>
-<span class="sourceLineNo">246</span> }<a name="line.246"></a>
-<span class="sourceLineNo">247</span> syncMethod.invoke(out, new Object[]{});<a name="line.247"></a>
-<span class="sourceLineNo">248</span> FSDataInputStream in = fs.open(p);<a name="line.248"></a>
-<span class="sourceLineNo">249</span> assertTrue(in.available() > 0);<a name="line.249"></a>
-<span class="sourceLineNo">250</span> byte [] buffer = new byte [1024];<a name="line.250"></a>
-<span class="sourceLineNo">251</span> int read = in.read(buffer);<a name="line.251"></a>
-<span class="sourceLineNo">252</span> assertEquals(tableName.getName().length, read);<a name="line.252"></a>
-<span class="sourceLineNo">253</span> out.close();<a name="line.253"></a>
-<span class="sourceLineNo">254</span> in.close();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> final int total = 20;<a name="line.256"></a>
-<span class="sourceLineNo">257</span> WAL.Reader reader = null;<a name="line.257"></a>
-<span class="sourceLineNo">258</span><a name="line.258"></a>
-<span class="sourceLineNo">259</span> try {<a name="line.259"></a>
-<span class="sourceLineNo">260</span> RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.260"></a>
-<span class="sourceLineNo">261</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.261"></a>
-<span class="sourceLineNo">262</span> scopes.put(tableName.getName(), 0);<a name="line.262"></a>
-<span class="sourceLineNo">263</span> final WAL wal = wals.getWAL(info);<a name="line.263"></a>
-<span class="sourceLineNo">264</span><a name="line.264"></a>
-<span class="sourceLineNo">265</span> for (int i = 0; i < total; i++) {<a name="line.265"></a>
-<span class="sourceLineNo">266</span> WALEdit kvs = new WALEdit();<a name="line.266"></a>
-<span class="sourceLineNo">267</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.267"></a>
-<span class="sourceLineNo">268</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.268"></a>
-<span class="sourceLineNo">269</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.269"></a>
-<span class="sourceLineNo">270</span> }<a name="line.270"></a>
-<span class="sourceLineNo">271</span> // Now call sync and try reading. Opening a Reader before you sync just<a name="line.271"></a>
-<span class="sourceLineNo">272</span> // gives you EOFE.<a name="line.272"></a>
-<span class="sourceLineNo">273</span> wal.sync();<a name="line.273"></a>
-<span class="sourceLineNo">274</span> // Open a Reader.<a name="line.274"></a>
-<span class="sourceLineNo">275</span> Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);<a name="line.275"></a>
-<span class="sourceLineNo">276</span> reader = wals.createReader(fs, walPath);<a name="line.276"></a>
-<span class="sourceLineNo">277</span> int count = 0;<a name="line.277"></a>
-<span class="sourceLineNo">278</span> WAL.Entry entry = new WAL.Entry();<a name="line.278"></a>
-<span class="sourceLineNo">279</span> while ((entry = reader.next(entry)) != null) count++;<a name="line.279"></a>
-<span class="sourceLineNo">280</span> assertEquals(total, count);<a name="line.280"></a>
-<span class="sourceLineNo">281</span> reader.close();<a name="line.281"></a>
-<span class="sourceLineNo">282</span> // Add test that checks to see that an open of a Reader works on a file<a name="line.282"></a>
-<span class="sourceLineNo">283</span> // that has had a sync done on it.<a name="line.283"></a>
-<span class="sourceLineNo">284</span> for (int i = 0; i < total; i++) {<a name="line.284"></a>
-<span class="sourceLineNo">285</span> WALEdit kvs = new WALEdit();<a name="line.285"></a>
-<span class="sourceLineNo">286</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.286"></a>
-<span class="sourceLineNo">287</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.287"></a>
-<span class="sourceLineNo">288</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.288"></a>
-<span class="sourceLineNo">289</span> }<a name="line.289"></a>
-<span class="sourceLineNo">290</span> wal.sync();<a name="line.290"></a>
-<span class="sourceLineNo">291</span> reader = wals.createReader(fs, walPath);<a name="line.291"></a>
-<span class="sourceLineNo">292</span> count = 0;<a name="line.292"></a>
-<span class="sourceLineNo">293</span> while((entry = reader.next(entry)) != null) count++;<a name="line.293"></a>
-<span class="sourceLineNo">294</span> assertTrue(count >= total);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> reader.close();<a name="line.295"></a>
-<span class="sourceLineNo">296</span> // If I sync, should see double the edits.<a name="line.296"></a>
-<span class="sourceLineNo">297</span> wal.sync();<a name="line.297"></a>
-<span class="sourceLineNo">298</span> reader = wals.createReader(fs, walPath);<a name="line.298"></a>
-<span class="sourceLineNo">299</span> count = 0;<a name="line.299"></a>
-<span class="sourceLineNo">300</span> while((entry = reader.next(entry)) != null) count++;<a name="line.300"></a>
-<span class="sourceLineNo">301</span> assertEquals(total * 2, count);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> reader.close();<a name="line.302"></a>
-<span class="sourceLineNo">303</span> // Now do a test that ensures stuff works when we go over block boundary,<a name="line.303"></a>
-<span class="sourceLineNo">304</span> // especially that we return good length on file.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> final byte [] value = new byte[1025 * 1024]; // Make a 1M value.<a name="line.305"></a>
-<span class="sourceLineNo">306</span> for (int i = 0; i < total; i++) {<a name="line.306"></a>
-<span class="sourceLineNo">307</span> WALEdit kvs = new WALEdit();<a name="line.307"></a>
-<span class="sourceLineNo">308</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value));<a name="line.308"></a>
-<span class="sourceLineNo">309</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.309"></a>
-<span class="sourceLineNo">310</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> }<a name="line.311"></a>
-<span class="sourceLineNo">312</span> // Now I should have written out lots of blocks. Sync then read.<a name="line.312"></a>
-<span class="sourceLineNo">313</span> wal.sync();<a name="line.313"></a>
-<span class="sourceLineNo">314</span> reader = wals.createReader(fs, walPath);<a name="line.314"></a>
-<span class="sourceLineNo">315</span> count = 0;<a name="line.315"></a>
-<span class="sourceLineNo">316</span> while((entry = reader.next(entry)) != null) count++;<a name="line.316"></a>
-<span class="sourceLineNo">317</span> assertEquals(total * 3, count);<a name="line.317"></a>
-<span class="sourceLineNo">318</span> reader.close();<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // shutdown and ensure that Reader gets right length also.<a name="line.319"></a>
-<span class="sourceLineNo">320</span> wal.shutdown();<a name="line.320"></a>
-<span class="sourceLineNo">321</span> reader = wals.createReader(fs, walPath);<a name="line.321"></a>
-<span class="sourceLineNo">322</span> count = 0;<a name="line.322"></a>
-<span class="sourceLineNo">323</span> while((entry = reader.next(entry)) != null) count++;<a name="line.323"></a>
-<span class="sourceLineNo">324</span> assertEquals(total * 3, count);<a name="line.324"></a>
-<span class="sourceLineNo">325</span> reader.close();<a name="line.325"></a>
-<span class="sourceLineNo">326</span> } finally {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> if (reader != null) reader.close();<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span> }<a name="line.329"></a>
-<span class="sourceLineNo">330</span><a name="line.330"></a>
-<span class="sourceLineNo">331</span> private void verifySplits(final List<Path> splits, final int howmany)<a name="line.331"></a>
-<span class="sourceLineNo">332</span> throws IOException {<a name="line.332"></a>
-<span class="sourceLineNo">333</span> assertEquals(howmany * howmany, splits.size());<a name="line.333"></a>
-<span class="sourceLineNo">334</span> for (int i = 0; i < splits.size(); i++) {<a name="line.334"></a>
-<span class="sourceLineNo">335</span> LOG.info("Verifying=" + splits.get(i));<a name="line.335"></a>
-<span class="sourceLineNo">336</span> WAL.Reader reader = wals.createReader(fs, splits.get(i));<a name="line.336"></a>
-<span class="sourceLineNo">337</span> try {<a name="line.337"></a>
-<span class="sourceLineNo">338</span> int count = 0;<a name="line.338"></a>
-<span class="sourceLineNo">339</span> String previousRegion = null;<a name="line.339"></a>
-<span class="sourceLineNo">340</span> long seqno = -1;<a name="line.340"></a>
-<span class="sourceLineNo">341</span> WAL.Entry entry = new WAL.Entry();<a name="line.341"></a>
-<span class="sourceLineNo">342</span> while((entry = reader.next(entry)) != null) {<a name="line.342"></a>
-<span class="sourceLineNo">343</span> WALKey key = entry.getKey();<a name="line.343"></a>
-<span class="sourceLineNo">344</span> String region = Bytes.toString(key.getEncodedRegionName());<a name="line.344"></a>
-<span class="sourceLineNo">345</span> // Assert that all edits are for same region.<a name="line.345"></a>
-<span class="sourceLineNo">346</span> if (previousRegion != null) {<a name="line.346"></a>
-<span class="sourceLineNo">347</span> assertEquals(previousRegion, region);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span> LOG.info("oldseqno=" + seqno + ", newseqno=" + key.getSequenceId());<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assertTrue(seqno < key.getSequenceId());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> seqno = key.getSequenceId();<a name="line.351"></a>
-<span class="sourceLineNo">352</span> previousRegion = region;<a name="line.352"></a>
-<span class="sourceLineNo">353</span> count++;<a name="line.353"></a>
-<span class="sourceLineNo">354</span> }<a name="line.354"></a>
-<span class="sourceLineNo">355</span> assertEquals(howmany, count);<a name="line.355"></a>
-<span class="sourceLineNo">356</span> } finally {<a name="line.356"></a>
-<span class="sourceLineNo">357</span> reader.close();<a name="line.357"></a>
-<span class="sourceLineNo">358</span> }<a name="line.358"></a>
-<span class="sourceLineNo">359</span> }<a name="line.359"></a>
-<span class="sourceLineNo">360</span> }<a name="line.360"></a>
-<span class="sourceLineNo">361</span><a name="line.361"></a>
-<span class="sourceLineNo">362</span> /*<a name="line.362"></a>
-<span class="sourceLineNo">363</span> * We pass different values to recoverFileLease() so that different code paths are covered<a name="line.363"></a>
-<span class="sourceLineNo">364</span> *<a name="line.364"></a>
-<span class="sourceLineNo">365</span> * For this test to pass, requires:<a name="line.365"></a>
-<span class="sourceLineNo">366</span> * 1. HDFS-200 (append support)<a name="line.366"></a>
-<span class="sourceLineNo">367</span> * 2. HDFS-988 (SafeMode should freeze file operations<a name="line.367"></a>
-<span class="sourceLineNo">368</span> * [FSNamesystem.nextGenerationStampForBlock])<a name="line.368"></a>
-<span class="sourceLineNo">369</span> * 3. HDFS-142 (on restart, maintain pendingCreates)<a name="line.369"></a>
-<span class="sourceLineNo">370</span> */<a name="line.370"></a>
-<span class="sourceLineNo">371</span> @Test<a name="line.371"></a>
-<span class="sourceLineNo">372</span> public void testAppendClose() throws Exception {<a name="line.372"></a>
-<span class="sourceLineNo">373</span> TableName tableName =<a name="line.373"></a>
-<span class="sourceLineNo">374</span> TableName.valueOf(currentTest.getMethodName());<a name="line.374"></a>
-<span class="sourceLineNo">375</span> RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.375"></a>
-<span class="sourceLineNo">376</span><a name="line.376"></a>
-<span class="sourceLineNo">377</span> WAL wal = wals.getWAL(regionInfo);<a name="line.377"></a>
-<span class="sourceLineNo">378</span> int total = 20;<a name="line.378"></a>
-<span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.380"></a>
-<span class="sourceLineNo">381</span> scopes.put(tableName.getName(), 0);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();<a name="line.382"></a>
-<span class="sourceLineNo">383</span> for (int i = 0; i < total; i++) {<a name="line.383"></a>
-<span class="sourceLineNo">384</span> WALEdit kvs = new WALEdit();<a name="line.384"></a>
-<span class="sourceLineNo">385</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.385"></a>
-<span class="sourceLineNo">386</span> wal.append(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,<a name="line.386"></a>
-<span class="sourceLineNo">387</span> System.currentTimeMillis(), mvcc, scopes),<a name="line.387"></a>
-<span class="sourceLineNo">388</span> kvs, true);<a name="line.388"></a>
-<span class="sourceLineNo">389</span> }<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // Now call sync to send the data to HDFS datanodes<a name="line.390"></a>
-<span class="sourceLineNo">391</span> wal.sync();<a name="line.391"></a>
-<span class="sourceLineNo">392</span> int namenodePort = cluster.getNameNodePort();<a name="line.392"></a>
-<span class="sourceLineNo">393</span> final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);<a name="line.393"></a>
-<span class="sourceLineNo">394</span><a name="line.394"></a>
-<span class="sourceLineNo">395</span><a name="line.395"></a>
-<span class="sourceLineNo">396</span> // Stop the cluster. (ensure restart since we're sharing MiniDFSCluster)<a name="line.396"></a>
-<span class="sourceLineNo">397</span> try {<a name="line.397"></a>
-<span class="sourceLineNo">398</span> DistributedFileSystem dfs = cluster.getFileSystem();<a name="line.398"></a>
-<span class="sourceLineNo">399</span> dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> TEST_UTIL.shutdownMiniDFSCluster();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> try {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> // wal.writer.close() will throw an exception,<a name="line.402"></a>
-<span class="sourceLineNo">403</span> // but still call this since it closes the LogSyncer thread first<a name="line.403"></a>
-<span class="sourceLineNo">404</span> wal.shutdown();<a name="line.404"></a>
-<span class="sourceLineNo">405</span> } catch (IOException e) {<a name="line.405"></a>
-<span class="sourceLineNo">406</span> LOG.info(e.toString(), e);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> }<a name="line.407"></a>
-<span class="sourceLineNo">408</span> fs.close(); // closing FS last so DFSOutputStream can't call close<a name="line.408"></a>
-<span class="sourceLineNo">409</span> LOG.info("STOPPED first instance of the cluster");<a name="line.409"></a>
-<span class="sourceLineNo">410</span> } finally {<a name="line.410"></a>
-<span class="sourceLineNo">411</span> // Restart the cluster<a name="line.411"></a>
-<span class="sourceLineNo">412</span> while (cluster.isClusterUp()){<a name="line.412"></a>
-<span class="sourceLineNo">413</span> LOG.error("Waiting for cluster to go down");<a name="line.413"></a>
-<span class="sourceLineNo">414</span> Thread.sleep(1000);<a name="line.414"></a>
-<span class="sourceLineNo">415</span> }<a name="line.415"></a>
-<span class="sourceLineNo">416</span> assertFalse(cluster.isClusterUp());<a name="line.416"></a>
-<span class="sourceLineNo">417</span> cluster = null;<a name="line.417"></a>
-<span class="sourceLineNo">418</span> for (int i = 0; i < 100; i++) {<a name="line.418"></a>
-<span class="sourceLineNo">419</span> try {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> cluster = TEST_UTIL.startMiniDFSClusterForTestWAL(namenodePort);<a name="line.420"></a>
-<span class="sourceLineNo">421</span> break;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> } catch (BindException e) {<a name="line.422"></a>
-<span class="sourceLineNo">423</span> LOG.info("Sleeping. BindException bringing up new cluster");<a name="line.423"></a>
-<span class="sourceLineNo">424</span> Threads.sleep(1000);<a name="line.424"></a>
-<span class="sourceLineNo">425</span> }<a name="line.425"></a>
-<span class="sourceLineNo">426</span> }<a name="line.426"></a>
-<span class="sourceLineNo">427</span> cluster.waitActive();<a name="line.427"></a>
-<span class="sourceLineNo">428</span> fs = cluster.getFileSystem();<a name="line.428"></a>
-<span class="sourceLineNo">429</span> LOG.info("STARTED second instance.");<a name="line.429"></a>
-<span class="sourceLineNo">430</span> }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span> // set the lease period to be 1 second so that the<a name="line.432"></a>
-<span class="sourceLineNo">433</span> // namenode triggers lease recovery upon append request<a name="line.433"></a>
-<span class="sourceLineNo">434</span> Method setLeasePeriod = cluster.getClass()<a name="line.434"></a>
-<span class="sourceLineNo">435</span> .getDeclaredMethod("setLeasePeriod", new Class[]{Long.TYPE, Long.TYPE});<a name="line.435"></a>
-<span class="sourceLineNo">436</span> setLeasePeriod.setAccessible(true);<a name="line.436"></a>
-<span class="sourceLineNo">437</span> setLeasePeriod.invoke(cluster, 1000L, 1000L);<a name="line.437"></a>
-<span class="sourceLineNo">438</span> try {<a name="line.438"></a>
-<span class="sourceLineNo">439</span> Thread.sleep(1000);<a name="line.439"></a>
-<span class="sourceLineNo">440</span> } catch (InterruptedException e) {<a name="line.440"></a>
-<span class="sourceLineNo">441</span> LOG.info(e.toString(), e);<a name="line.441"></a>
-<span class="sourceLineNo">442</span> }<a name="line.442"></a>
-<span class="sourceLineNo">443</span><a name="line.443"></a>
-<span class="sourceLineNo">444</span> // Now try recovering the log, like the HMaster would do<a name="line.444"></a>
-<span class="sourceLineNo">445</span> final FileSystem recoveredFs = fs;<a name="line.445"></a>
-<span class="sourceLineNo">446</span> final Configuration rlConf = conf;<a name="line.446"></a>
-<span class="sourceLineNo">447</span><a name="line.447"></a>
-<span class="sourceLineNo">448</span> class RecoverLogThread extends Thread {<a name="line.448"></a>
-<span class="sourceLineNo">449</span> public Exception exception = null;<a name="line.449"></a>
-<span class="sourceLineNo">450</span> @Override<a name="line.450"></a>
-<span class="sourceLineNo">451</span> public void run() {<a name="line.451"></a>
-<span class="sourceLineNo">452</span> try {<a name="line.452"></a>
-<span class="sourceLineNo">453</span> FSUtils.getInstance(fs, rlConf)<a name="line.453"></a>
-<span class="sourceLineNo">454</span> .recoverFileLease(recoveredFs, walPath, rlConf, null);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> } catch (IOException e) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span> exception = e;<a name="line.456"></a>
-<span class="sourceLineNo">457</span> }<a name="line.457"></a>
-<span class="sourceLineNo">458</span> }<a name="line.458"></a>
-<span class="sourceLineNo">459</span> }<a name="line.459"></a>
-<span class="sourceLineNo">460</span><a name="line.460"></a>
-<span class="sourceLineNo">461</span> RecoverLogThread t = new RecoverLogThread();<a name="line.461"></a>
-<span class="sourceLineNo">462</span> t.start();<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // Timeout after 60 sec. Without correct patches, would be an infinite loop<a name="line.463"></a>
-<span class="sourceLineNo">464</span> t.join(60 * 1000);<a name="line.464"></a>
-<span class="sourceLineNo">465</span> if(t.isAlive()) {<a name="line.465"></a>
-<span class="sourceLineNo">466</span> t.interrupt();<a name="line.466"></a>
-<span class="sourceLineNo">467</span> throw new Exception("Timed out waiting for WAL.recoverLog()");<a name="line.467"></a>
-<span class="sourceLineNo">468</span> }<a name="line.468"></a>
-<span class="sourceLineNo">469</span><a name="line.469"></a>
-<span class="sourceLineNo">470</span> if (t.exception != null)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throw t.exception;<a name="line.471"></a>
-<span class="sourceLineNo">472</span><a name="line.472"></a>
-<span class="sourceLineNo">473</span> // Make sure you can read all the content<a name="line.473"></a>
-<span class="sourceLineNo">474</span> WAL.Reader reader = wals.createReader(fs, walPath);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> int count = 0;<a name="line.475"></a>
-<span class="sourceLineNo">476</span> WAL.Entry entry = new WAL.Entry();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> while (reader.next(entry) != null) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> count++;<a name="line.478"></a>
-<span class="sourceLineNo">479</span> assertTrue("Should be one KeyValue per WALEdit",<a name="line.479"></a>
-<span class="sourceLineNo">480</span> entry.getEdit().getCells().size() == 1);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> }<a name="line.481"></a>
-<span class="sourceLineNo">482</span> assertEquals(total, count);<a name="line.482"></a>
-<span class="sourceLineNo">483</span> reader.close();<a name="line.483"></a>
-<span class="sourceLineNo">484</span><a name="line.484"></a>
-<span class="sourceLineNo">485</span> // Reset the lease period<a name="line.485"></a>
-<span class="sourceLineNo">486</span> setLeasePeriod.invoke(cluster, new Object[]{ 60000L, 3600000L });<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span><a name="line.488"></a>
-<span class="sourceLineNo">489</span> /**<a name="line.489"></a>
-<span class="sourceLineNo">490</span> * Tests that we can write out an edit, close, and then read it back in again.<a name="line.490"></a>
-<span class="sourceLineNo">491</span> */<a name="line.491"></a>
-<span class="sourceLineNo">492</span> @Test<a name="line.492"></a>
-<span class="sourceLineNo">493</span> public void testEditAdd() throws IOException {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> int colCount = 10;<a name="line.494"></a>
-<span class="sourceLineNo">495</span> TableDescriptor htd =<a name="line.495"></a>
-<span class="sourceLineNo">496</span> TableDescriptorBuilder.newBuilder(TableName.valueOf(currentTest.getMethodName()))<a name="line.496"></a>
-<span class="sourceLineNo">497</span> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("column")).build();<a name="line.497"></a>
-<span class="sourceLineNo">498</span> NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);<a name="line.498"></a>
-<span class="sourceLineNo">499</span> for (byte[] fam : htd.getColumnFamilyNames()) {<a name="line.499"></a>
-<span class="sourceLineNo">500</span> scopes.put(fam, 0);<a name="line.500"></a>
-<span class="sourceLineNo">501</span> }<a name="line.501"></a>
-<span class="sourceLineNo">502</span> byte[] row = Bytes.toBytes("row");<a name="line.502"></a>
-<span class="sourceLineNo">503</span> WAL.Reader reader = null;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> try {<a name="line.504"></a>
-<span class="sourceLineNo">505</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.505"></a>
-<span class="sourceLineNo">506</span><a name="line.506"></a>
-<span class="sourceLineNo">507</span> // Write columns named 1, 2, 3, etc. and then values of single byte<a name="line.507"></a>
-<span class="sourceLineNo">508</span> // 1, 2, 3...<a name="line.508"></a>
-<span class="sourceLineNo">509</span> long timestamp = System.currentTimeMillis();<a name="line.509"></a>
-<span class="sourceLineNo">510</span> WALEdit cols = new WALEdit();<a name="line.510"></a>
-<span class="sourceLineNo">511</span> for (int i = 0; i < colCount; i++) {<a name="line.511"></a>
-<span class="sourceLineNo">512</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.512"></a>
-<span class="sourceLineNo">513</span> Bytes.toBytes(Integer.toString(i)),<a name="line.513"></a>
-<span class="sourceLineNo">514</span> timestamp, new byte[] { (byte)(i + '0') }));<a name="line.514"></a>
-<span class="sourceLineNo">515</span> }<a name="line.515"></a>
-<span class="sourceLineNo">516</span> RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(row)<a name="line.516"></a>
-<span class="sourceLineNo">517</span> .setEndKey(Bytes.toBytes(Bytes.toString(row) + "1")).build();<a name="line.517"></a>
-<span class="sourceLineNo">518</span> final WAL log = wals.getWAL(info);<a name="line.518"></a>
-<span class="sourceLineNo">519</span><a name="line.519"></a>
-<span class="sourceLineNo">520</span> final long txid = log.append(info,<a name="line.520"></a>
-<span class="sourceLineNo">521</span> new WALKeyImpl(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),<a name="line.521"></a>
-<span class="sourceLineNo">522</span> mvcc, scopes),<a name="line.522"></a>
-<span class="sourceLineNo">523</span> cols, true);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> log.sync(txid);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> log.startCacheFlush(info.getEncodedNameAsBytes(), htd.getColumnFamilyNames());<a name="line.525"></a>
-<span class="sourceLineNo">526</span> log.completeCacheFlush(info.getEncodedNameAsBytes());<a name="line.526"></a>
-<span class="sourceLineNo">527</span> log.shutdown();<a name="line.527"></a>
-<span class="sourceLineNo">528</span> Path filename = AbstractFSWALProvider.getCurrentFileName(log);<a name="line.528"></a>
-<span class="sourceLineNo">529</span> // Now open a reader on the log and assert append worked.<a name="line.529"></a>
-<span class="sourceLineNo">530</span> reader = wals.createReader(fs, filename);<a name="line.530"></a>
-<span class="sourceLineNo">531</span> // Above we added all columns on a single row so we only read one<a name="line.531"></a>
-<span class="sourceLineNo">532</span> // entry in the below... thats why we have '1'.<a name="line.532"></a>
-<span class="sourceLineNo">533</span> for (int i = 0; i < 1; i++) {<a name="line.533"></a>
-<span class="sourceLineNo">534</span> WAL.Entry entry = reader.next(null);<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (entry == null) break;<a name="line.535"></a>
-<span class="sourceLineNo">536</span> WALKey key = entry.getKey();<a name="line.536"></a>
-<span class="sourceLineNo">537</span> WALEdit val = entry.getEdit();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));<a name="line.538"></a>
-<span class="sourceLineNo">539</span> assertTrue(htd.getTableName().equals(key.getTableName()));<a name="line.539"></a>
-<span class="sourceLineNo">540</span> Cell cell = val.getCells().get(0);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> assertTrue(Bytes.equals(row, 0, row.length, cell.getRowArray(), cell.getRowOffset(),<a name="line.541"></a>
-<span class="sourceLineNo">542</span> cell.getRowLength()));<a name="line.542"></a>
-<span class="sourceLineNo">543</span> assertEquals((byte)(i + '0'), CellUtil.cloneValue(cell)[0]);<a name="line.543"></a>
-<span class="sourceLineNo">544</span> System.out.println(key + " " + val);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> } finally {<a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (reader != null) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> reader.close();<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span> }<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> @Test<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public void testAppend() throws IOException {<a name="line.554"></a>
-<span class="sourceLineNo">555</span> int colCount = 10;<a name="line.555"></a>
-<span class="sourceLineNo">556</span> TableDescriptor htd =<a name="line.556"></a>
-<span class="sourceLineNo">557</span> TableDescriptorBuilder.newBuilder(TableName.valueOf(currentTest.getMethodName()))<a name="line.557"></a>
-<span class="sourceLineNo">558</span> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("column")).build();<a name="line.558"></a>
-<span class="sourceLineNo">559</span> NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);<a name="line.559"></a>
-<span class="sourceLineNo">560</span> for (byte[] fam : htd.getColumnFamilyNames()) {<a name="line.560"></a>
-<span class="sourceLineNo">561</span> scopes.put(fam, 0);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> }<a name="line.562"></a>
-<span class="sourceLineNo">563</span> byte[] row = Bytes.toBytes("row");<a name="line.563"></a>
-<span class="sourceLineNo">564</span> WAL.Reader reader = null;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.565"></a>
-<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // Write columns named 1, 2, 3, etc. and then values of single byte<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // 1, 2, 3...<a name="line.568"></a>
-<span class="sourceLineNo">569</span> long timestamp = System.currentTimeMillis();<a name="line.569"></a>
-<span class="sourceLineNo">570</span> WALEdit cols = new WALEdit();<a name="line.570"></a>
-<span class="sourceLineNo">571</span> for (int i = 0; i < colCount; i++) {<a name="line.571"></a>
-<span class="sourceLineNo">572</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.572"></a>
-<span class="sourceLineNo">573</span> Bytes.toBytes(Integer.toString(i)),<a name="line.573"></a>
-<span class="sourceLineNo">574</span> timestamp, new byte[] { (byte)(i + '0') }));<a name="line.574"></a>
-<span class="sourceLineNo">575</span> }<a name="line.575"></a>
-<span class="sourceLineNo">576</span> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();<a name="line.576"></a>
-<span class="sourceLineNo">577</span> final WAL log = wals.getWAL(hri);<a name="line.577"></a>
-<span class="sourceLineNo">578</span> final long txid = log.append(hri,<a name="line.578"></a>
-<span class="sourceLineNo">579</span> new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),<a name="line.579"></a>
-<span class="sourceLineNo">580</span> mvcc, scopes),<a name="line.580"></a>
-<span class="sourceLineNo">581</span> cols, true);<a name="line.581"></a>
-<span class="sourceLineNo">582</span> log.sync(txid);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getColumnFamilyNames());<a name="line.583"></a>
-<span class="sourceLineNo">584</span> log.completeCacheFlush(hri.getEncodedNameAsBytes());<a name="line.584"></a>
-<span class="sourceLineNo">585</span> log.shutdown();<a name="line.585"></a>
-<span class="sourceLineNo">586</span> Path filename = AbstractFSWALProvider.getCurrentFileName(log);<a name="line.586"></a>
-<span class="sourceLineNo">587</span> // Now open a reader on the log and assert append worked.<a name="line.587"></a>
-<span class="sourceLineNo">588</span> reader = wals.createReader(fs, filename);<a name="line.588"></a>
-<span class="sourceLineNo">589</span> WAL.Entry entry = reader.next();<a name="line.589"></a>
-<span class="sourceLineNo">590</span> assertEquals(colCount, entry.getEdit().size());<a name="line.590"></a>
-<span class="sourceLineNo">591</span> int idx = 0;<a name="line.591"></a>
-<span class="sourceLineNo">592</span> for (Cell val : entry.getEdit().getCells()) {<a name="line.592"></a>
-<span class="sourceLineNo">593</span> assertTrue(Bytes.equals(hri.getEncodedNameAsBytes(),<a name="line.593"></a>
-<span class="sourceLineNo">594</span> entry.getKey().getEncodedRegionName()));<a name="line.594"></a>
-<span class="sourceLineNo">595</span> assertTrue(htd.getTableName().equals(entry.getKey().getTableName()));<a name="line.595"></a>
-<span class="sourceLineNo">596</span> assertTrue(Bytes.equals(row, 0, row.length, val.getRowArray(), val.getRowOffset(),<a name="line.596"></a>
-<span class="sourceLineNo">597</span> val.getRowLength()));<a name="line.597"></a>
-<span class="sourceLineNo">598</span> assertEquals((byte) (idx + '0'), CellUtil.cloneValue(val)[0]);<a name="line.598"></a>
-<span class="sourceLineNo">599</span> System.out.println(entry.getKey() + " " + val);<a name="line.599"></a>
-<span class="sourceLineNo">600</span> idx++;<a name="line.600"></a>
-<span class="sourceLineNo">601</span> }<a name="line.601"></a>
-<span class="sourceLineNo">602</span> } finally {<a name="line.602"></a>
-<span class="sourceLineNo">603</span> if (reader != null) {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> reader.close();<a name="line.604"></a>
-<span class="sourceLineNo">605</span> }<a name="line.605"></a>
-<span class="sourceLineNo">606</span> }<a name="line.606"></a>
-<span class="sourceLineNo">607</span> }<a name="line.607"></a>
-<span class="sourceLineNo">608</span><a name="line.608"></a>
-<span class="sourceLineNo">609</span> /**<a name="line.609"></a>
-<span class="sourceLineNo">610</span> * Test that we can visit entries before they are appended<a name="line.610"></a>
-<span class="sourceLineNo">611</span> * @throws Exception<a name="line.611"></a>
-<span class="sourceLineNo">612</span> */<a name="line.612"></a>
-<span class="sourceLineNo">613</span> @Test<a name="line.613"></a>
-<span class="sourceLineNo">614</span> public void testVisitors() throws Exception {<a name="line.614"></a>
-<span class="sourceLineNo">615</span> final int COL_COUNT = 10;<a name="line.615"></a>
-<span class="sourceLineNo">616</span> final TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.616"></a>
-<span class="sourceLineNo">617</span> final byte [] row = Bytes.toBytes("row");<a name="line.617"></a>
-<span class="sourceLineNo">618</span> final DumbWALActionsListener visitor = new DumbWALActionsListener();<a name="line.618"></a>
-<span class="sourceLineNo">619</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.619"></a>
-<span class="sourceLineNo">620</span> long timestamp = System.currentTimeMillis();<a name="line.620"></a>
-<span class="sourceLineNo">621</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.621"></a>
-<span class="sourceLineNo">622</span> scopes.put(Bytes.toBytes("column"), 0);<a name="line.622"></a>
-<span class="sourceLineNo">623</span><a name="line.623"></a>
-<span class="sourceLineNo">624</span> RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.624"></a>
-<span class="sourceLineNo">625</span> final WAL log = wals.getWAL(hri);<a name="line.625"></a>
-<span class="sourceLineNo">626</span> log.registerWALActionsListener(visitor);<a name="line.626"></a>
-<span class="sourceLineNo">627</span> for (int i = 0; i < COL_COUNT; i++) {<a name="line.627"></a>
-<span class="sourceLineNo">628</span> WALEdit cols = new WALEdit();<a name="line.628"></a>
-<span class="sourceLineNo">629</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.629"></a>
-<span class="sourceLineNo">630</span> Bytes.toBytes(Integer.toString(i)),<a name="line.630"></a>
-<span class="sourceLineNo">631</span> timestamp, new byte[]{(byte) (i + '0')}));<a name="line.631"></a>
-<span class="sourceLineNo">632</span> log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName,<a name="line.632"></a>
-<span class="sourceLineNo">633</span> System.currentTimeMillis(), mvcc, scopes), cols, true);<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> log.sync();<a name="line.635"></a>
-<span class="sourceLineNo">636</span> assertEquals(COL_COUNT, visitor.increments);<a name="line.636"></a>
-<span class="sourceLineNo">637</span> log.unregisterWALActionsListener(visitor);<a name="line.637"></a>
-<span class="sourceLineNo">638</span> WALEdit cols = new WALEdit();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.639"></a>
-<span class="sourceLineNo">640</span> Bytes.toBytes(Integer.toString(11)),<a name="line.640"></a>
-<span class="sourceLineNo">641</span> timestamp, new byte[]{(byte) (11 + '0')}));<a name="line.641"></a>
-<span class="sourceLineNo">642</span> log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName,<a name="line.642"></a>
-<span class="sourceLineNo">643</span> System.currentTimeMillis(), mvcc, scopes), cols, true);<a name="line.643"></a>
-<span class="sourceLineNo">644</span> log.sync();<a name="line.644"></a>
-<span class="sourceLineNo">645</span> assertEquals(COL_COUNT, visitor.increments);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span> /**<a name="line.648"></a>
-<span class="sourceLineNo">649</span> * A loaded WAL coprocessor won't break existing WAL test cases.<a name="line.649"></a>
-<span class="sourceLineNo">650</span> */<a name="line.650"></a>
-<span class="sourceLineNo">651</span> @Test<a name="line.651"></a>
-<span class="sourceLineNo">652</span> public void testWALCoprocessorLoaded() throws Exception {<a name="line.652"></a>
-<span class="sourceLineNo">653</span> // test to see whether the coprocessor is loaded or not.<a name="line.653"></a>
-<span class="sourceLineNo">654</span> WALCoprocessorHost host = wals.getWAL(null).getCoprocessorHost();<a name="line.654"></a>
-<span class="sourceLineNo">655</span> Coprocessor c = host.findCoprocessor(SampleRegionWALCoprocessor.class);<a name="line.655"></a>
-<span class="sourceLineNo">656</span> assertNotNull(c);<a name="line.656"></a>
-<span class="sourceLineNo">657</span> }<a name="line.657"></a>
-<span class="sourceLineNo">658</span><a name="line.658"></a>
-<span class="sourceLineNo">659</span> static class DumbWALActionsListener implements WALActionsListener {<a name="line.659"></a>
-<span class="sourceLineNo">660</span> int increments = 0;<a name="line.660"></a>
-<span class="sourceLineNo">661</span><a name="line.661"></a>
-<span class="sourceLineNo">662</span> @Override<a name="line.662"></a>
-<span class="sourceLineNo">663</span> public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {<a name="line.663"></a>
-<span class="sourceLineNo">664</span> increments++;<a name="line.664"></a>
-<span class="sourceLineNo">665</span> }<a name="line.665"></a>
-<span class="sourceLineNo">666</span><a name="line.666"></a>
-<span class="sourceLineNo">667</span> @Override<a name="line.667"></a>
-<span class="sourceLineNo">668</span> public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // To change body of implemented methods use File | Settings | File<a name="line.669"></a>
-<span class="sourceLineNo">670</span> // Templates.<a name="line.670"></a>
-<span class="sourceLineNo">671</span> increments++;<a name="line.671"></a>
-<span class="sourceLineNo">672</span> }<a name="line.672"></a>
-<span class="sourceLineNo">673</span> }<a name="line.673"></a>
-<span class="sourceLineNo">674</span>}<a name="line.674"></a>
+<span class="sourceLineNo">143</span> "dfs.client.block.recovery.retries", 1);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> TEST_UTIL.getConfiguration().setInt(<a name="line.144"></a>
+<span class="sourceLineNo">145</span> "hbase.ipc.client.connection.maxidletime", 500);<a name="line.145"></a>
+<span class="sourceLineNo">146</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.timeout", 10000);<a name="line.146"></a>
+<span class="sourceLineNo">147</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 1000);<a name="line.147"></a>
+<span class="sourceLineNo">148</span> TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,<a name="line.148"></a>
+<span class="sourceLineNo">149</span> SampleRegionWALCoprocessor.class.getName());<a name="line.149"></a>
+<span class="sourceLineNo">150</span> TEST_UTIL.startMiniDFSCluster(3);<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span> conf = TEST_UTIL.getConfiguration();<a name="line.152"></a>
+<span class="sourceLineNo">153</span> cluster = TEST_UTIL.getDFSCluster();<a name="line.153"></a>
+<span class="sourceLineNo">154</span><a name="line.154"></a>
+<span class="sourceLineNo">155</span> hbaseDir = TEST_UTIL.createRootDir();<a name="line.155"></a>
+<span class="sourceLineNo">156</span> hbaseWALDir = TEST_UTIL.createWALRootDir();<a name="line.156"></a>
+<span class="sourceLineNo">157</span> }<a name="line.157"></a>
+<span class="sourceLineNo">158</span><a name="line.158"></a>
+<span class="sourceLineNo">159</span> @AfterClass<a name="line.159"></a>
+<span class="sourceLineNo">160</span> public static void tearDownAfterClass() throws Exception {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> TEST_UTIL.shutdownMiniCluster();<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
+<span class="sourceLineNo">163</span><a name="line.163"></a>
+<span class="sourceLineNo">164</span> @Test<a name="line.164"></a>
+<span class="sourceLineNo">165</span> public void canCloseSingleton() throws IOException {<a name="line.165"></a>
+<span class="sourceLineNo">166</span> WALFactory.getInstance(conf).close();<a name="line.166"></a>
+<span class="sourceLineNo">167</span> }<a name="line.167"></a>
+<span class="sourceLineNo">168</span><a name="line.168"></a>
+<span class="sourceLineNo">169</span> /**<a name="line.169"></a>
+<span class="sourceLineNo">170</span> * Just write multiple logs then split. Before fix for HADOOP-2283, this<a name="line.170"></a>
+<span class="sourceLineNo">171</span> * would fail.<a name="line.171"></a>
+<span class="sourceLineNo">172</span> * @throws IOException<a name="line.172"></a>
+<span class="sourceLineNo">173</span> */<a name="line.173"></a>
+<span class="sourceLineNo">174</span> @Test<a name="line.174"></a>
+<span class="sourceLineNo">175</span> public void testSplit() throws IOException {<a name="line.175"></a>
+<span class="sourceLineNo">176</span> final TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.176"></a>
+<span class="sourceLineNo">177</span> final byte [] rowName = tableName.getName();<a name="line.177"></a>
+<span class="sourceLineNo">178</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.178"></a>
+<span class="sourceLineNo">179</span> final int howmany = 3;<a name="line.179"></a>
+<span class="sourceLineNo">180</span> RegionInfo[] infos = new RegionInfo[3];<a name="line.180"></a>
+<span class="sourceLineNo">181</span> Path tabledir = FSUtils.getTableDir(hbaseDir, tableName);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> fs.mkdirs(tabledir);<a name="line.182"></a>
+<span class="sourceLineNo">183</span> for (int i = 0; i < howmany; i++) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span> infos[i] = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("" + i))<a name="line.184"></a>
+<span class="sourceLineNo">185</span> .setEndKey(Bytes.toBytes("" + (i + 1))).build();<a name="line.185"></a>
+<span class="sourceLineNo">186</span> fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));<a name="line.186"></a>
+<span class="sourceLineNo">187</span> LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());<a name="line.187"></a>
+<span class="sourceLineNo">188</span> }<a name="line.188"></a>
+<span class="sourceLineNo">189</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.189"></a>
+<span class="sourceLineNo">190</span> scopes.put(Bytes.toBytes("column"), 0);<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span> // Add edits for three regions.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> for (int ii = 0; ii < howmany; ii++) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span> for (int i = 0; i < howmany; i++) {<a name="line.195"></a>
+<span class="sourceLineNo">196</span> final WAL log =<a name="line.196"></a>
+<span class="sourceLineNo">197</span> wals.getWAL(infos[i]);<a name="line.197"></a>
+<span class="sourceLineNo">198</span> for (int j = 0; j < howmany; j++) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span> WALEdit edit = new WALEdit();<a name="line.199"></a>
+<span class="sourceLineNo">200</span> byte [] family = Bytes.toBytes("column");<a name="line.200"></a>
+<span class="sourceLineNo">201</span> byte [] qualifier = Bytes.toBytes(Integer.toString(j));<a name="line.201"></a>
+<span class="sourceLineNo">202</span> byte [] column = Bytes.toBytes("column:" + Integer.toString(j));<a name="line.202"></a>
+<span class="sourceLineNo">203</span> edit.add(new KeyValue(rowName, family, qualifier,<a name="line.203"></a>
+<span class="sourceLineNo">204</span> System.currentTimeMillis(), column));<a name="line.204"></a>
+<span class="sourceLineNo">205</span> LOG.info("Region " + i + ": " + edit);<a name="line.205"></a>
+<span class="sourceLineNo">206</span> WALKeyImpl walKey = new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName,<a name="line.206"></a>
+<span class="sourceLineNo">207</span> System.currentTimeMillis(), mvcc, scopes);<a name="line.207"></a>
+<span class="sourceLineNo">208</span> log.append(infos[i], walKey, edit, true);<a name="line.208"></a>
+<span class="sourceLineNo">209</span> walKey.getWriteEntry();<a name="line.209"></a>
+<span class="sourceLineNo">210</span> }<a name="line.210"></a>
+<span class="sourceLineNo">211</span> log.sync();<a name="line.211"></a>
+<span class="sourceLineNo">212</span> log.rollWriter(true);<a name="line.212"></a>
+<span class="sourceLineNo">213</span> }<a name="line.213"></a>
+<span class="sourceLineNo">214</span> }<a name="line.214"></a>
+<span class="sourceLineNo">215</span> wals.shutdown();<a name="line.215"></a>
+<span class="sourceLineNo">216</span> // The below calculation of logDir relies on insider information... WALSplitter should be connected better<a name="line.216"></a>
+<span class="sourceLineNo">217</span> // with the WAL system.... not requiring explicit path. The oldLogDir is just made up not used.<a name="line.217"></a>
+<span class="sourceLineNo">218</span> Path logDir =<a name="line.218"></a>
+<span class="sourceLineNo">219</span> new Path(new Path(hbaseWALDir, HConstants.HREGION_LOGDIR_NAME),<a name="line.219"></a>
+<span class="sourceLineNo">220</span> this.currentServername.toString());<a name="line.220"></a>
+<span class="sourceLineNo">221</span> Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.221"></a>
+<span class="sourceLineNo">222</span> List<Path> splits = WALSplitter.split(hbaseWALDir, logDir, oldLogDir, fs, conf, wals);<a name="line.222"></a>
+<span class="sourceLineNo">223</span> verifySplits(splits, howmany);<a name="line.223"></a>
+<span class="sourceLineNo">224</span> }<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span> /**<a name="line.226"></a>
+<span class="sourceLineNo">227</span> * Test new HDFS-265 sync.<a name="line.227"></a>
+<span class="sourceLineNo">228</span> * @throws Exception<a name="line.228"></a>
+<span class="sourceLineNo">229</span> */<a name="line.229"></a>
+<span class="sourceLineNo">230</span> @Test<a name="line.230"></a>
+<span class="sourceLineNo">231</span> public void Broken_testSync() throws Exception {<a name="line.231"></a>
+<span class="sourceLineNo">232</span> TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.232"></a>
+<span class="sourceLineNo">233</span> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.233"></a>
+<span class="sourceLineNo">234</span> // First verify that using streams all works.<a name="line.234"></a>
+<span class="sourceLineNo">235</span> Path p = new Path(dir, currentTest.getMethodName() + ".fsdos");<a name="line.235"></a>
+<span class="sourceLineNo">236</span> FSDataOutputStream out = fs.create(p);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> out.write(tableName.getName());<a name="line.237"></a>
+<span class="sourceLineNo">238</span> Method syncMethod = null;<a name="line.238"></a>
+<span class="sourceLineNo">239</span> try {<a name="line.239"></a>
+<span class="sourceLineNo">240</span> syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});<a name="line.240"></a>
+<span class="sourceLineNo">241</span> } catch (NoSuchMethodException e) {<a name="line.241"></a>
+<span class="sourceLineNo">242</span> try {<a name="line.242"></a>
+<span class="sourceLineNo">243</span> syncMethod = out.getClass().getMethod("sync", new Class<?> []{});<a name="line.243"></a>
+<span class="sourceLineNo">244</span> } catch (NoSuchMethodException ex) {<a name="line.244"></a>
+<span class="sourceLineNo">245</span> fail("This version of Hadoop supports neither Syncable.sync() " +<a name="line.245"></a>
+<span class="sourceLineNo">246</span> "nor Syncable.hflush().");<a name="line.246"></a>
+<span class="sourceLineNo">247</span> }<a name="line.247"></a>
+<span class="sourceLineNo">248</span> }<a name="line.248"></a>
+<span class="sourceLineNo">249</span> syncMethod.invoke(out, new Object[]{});<a name="line.249"></a>
+<span class="sourceLineNo">250</span> FSDataInputStream in = fs.open(p);<a name="line.250"></a>
+<span class="sourceLineNo">251</span> assertTrue(in.available() > 0);<a name="line.251"></a>
+<span class="sourceLineNo">252</span> byte [] buffer = new byte [1024];<a name="line.252"></a>
+<span class="sourceLineNo">253</span> int read = in.read(buffer);<a name="line.253"></a>
+<span class="sourceLineNo">254</span> assertEquals(tableName.getName().length, read);<a name="line.254"></a>
+<span class="sourceLineNo">255</span> out.close();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> in.close();<a name="line.256"></a>
+<span class="sourceLineNo">257</span><a name="line.257"></a>
+<span class="sourceLineNo">258</span> final int total = 20;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> WAL.Reader reader = null;<a name="line.259"></a>
+<span class="sourceLineNo">260</span><a name="line.260"></a>
+<span class="sourceLineNo">261</span> try {<a name="line.261"></a>
+<span class="sourceLineNo">262</span> RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.262"></a>
+<span class="sourceLineNo">263</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> scopes.put(tableName.getName(), 0);<a name="line.264"></a>
+<span class="sourceLineNo">265</span> final WAL wal = wals.getWAL(info);<a name="line.265"></a>
+<span class="sourceLineNo">266</span><a name="line.266"></a>
+<span class="sourceLineNo">267</span> for (int i = 0; i < total; i++) {<a name="line.267"></a>
+<span class="sourceLineNo">268</span> WALEdit kvs = new WALEdit();<a name="line.268"></a>
+<span class="sourceLineNo">269</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.269"></a>
+<span class="sourceLineNo">270</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.270"></a>
+<span class="sourceLineNo">271</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> // Now call sync and try reading. Opening a Reader before you sync just<a name="line.273"></a>
+<span class="sourceLineNo">274</span> // gives you EOFE.<a name="line.274"></a>
+<span class="sourceLineNo">275</span> wal.sync();<a name="line.275"></a>
+<span class="sourceLineNo">276</span> // Open a Reader.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);<a name="line.277"></a>
+<span class="sourceLineNo">278</span> reader = wals.createReader(fs, walPath);<a name="line.278"></a>
+<span class="sourceLineNo">279</span> int count = 0;<a name="line.279"></a>
+<span class="sourceLineNo">280</span> WAL.Entry entry = new WAL.Entry();<a name="line.280"></a>
+<span class="sourceLineNo">281</span> while ((entry = reader.next(entry)) != null) count++;<a name="line.281"></a>
+<span class="sourceLineNo">282</span> assertEquals(total, count);<a name="line.282"></a>
+<span class="sourceLineNo">283</span> reader.close();<a name="line.283"></a>
+<span class="sourceLineNo">284</span> // Add test that checks to see that an open of a Reader works on a file<a name="line.284"></a>
+<span class="sourceLineNo">285</span> // that has had a sync done on it.<a name="line.285"></a>
+<span class="sourceLineNo">286</span> for (int i = 0; i < total; i++) {<a name="line.286"></a>
+<span class="sourceLineNo">287</span> WALEdit kvs = new WALEdit();<a name="line.287"></a>
+<span class="sourceLineNo">288</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.288"></a>
+<span class="sourceLineNo">289</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.289"></a>
+<span class="sourceLineNo">290</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.290"></a>
+<span class="sourceLineNo">291</span> }<a name="line.291"></a>
+<span class="sourceLineNo">292</span> wal.sync();<a name="line.292"></a>
+<span class="sourceLineNo">293</span> reader = wals.createReader(fs, walPath);<a name="line.293"></a>
+<span class="sourceLineNo">294</span> count = 0;<a name="line.294"></a>
+<span class="sourceLineNo">295</span> while((entry = reader.next(entry)) != null) count++;<a name="line.295"></a>
+<span class="sourceLineNo">296</span> assertTrue(count >= total);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> reader.close();<a name="line.297"></a>
+<span class="sourceLineNo">298</span> // If I sync, should see double the edits.<a name="line.298"></a>
+<span class="sourceLineNo">299</span> wal.sync();<a name="line.299"></a>
+<span class="sourceLineNo">300</span> reader = wals.createReader(fs, walPath);<a name="line.300"></a>
+<span class="sourceLineNo">301</span> count = 0;<a name="line.301"></a>
+<span class="sourceLineNo">302</span> while((entry = reader.next(entry)) != null) count++;<a name="line.302"></a>
+<span class="sourceLineNo">303</span> assertEquals(total * 2, count);<a name="line.303"></a>
+<span class="sourceLineNo">304</span> reader.close();<a name="line.304"></a>
+<span class="sourceLineNo">305</span> // Now do a test that ensures stuff works when we go over block boundary,<a name="line.305"></a>
+<span class="sourceLineNo">306</span> // especially that we return good length on file.<a name="line.306"></a>
+<span class="sourceLineNo">307</span> final byte [] value = new byte[1025 * 1024]; // Make a 1M value.<a name="line.307"></a>
+<span class="sourceLineNo">308</span> for (int i = 0; i < total; i++) {<a name="line.308"></a>
+<span class="sourceLineNo">309</span> WALEdit kvs = new WALEdit();<a name="line.309"></a>
+<span class="sourceLineNo">310</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value));<a name="line.310"></a>
+<span class="sourceLineNo">311</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.311"></a>
+<span class="sourceLineNo">312</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.312"></a>
+<span class="sourceLineNo">313</span> }<a name="line.313"></a>
+<span class="sourceLineNo">314</span> // Now I should have written out lots of blocks. Sync then read.<a name="line.314"></a>
+<span class="sourceLineNo">315</span> wal.sync();<a name="line.315"></a>
+<span class="sourceLineNo">316</span> reader = wals.createReader(fs, walPath);<a name="line.316"></a>
+<span class="sourceLineNo">317</span> count = 0;<a name="line.317"></a>
+<span class="sourceLineNo">318</span> while((entry = reader.next(entry)) != null) count++;<a name="line.318"></a>
+<span class="sourceLineNo">319</span> assertEquals(total * 3, count);<a name="line.319"></a>
+<span class="sourceLineNo">320</span> reader.close();<a name="line.320"></a>
+<span class="sourceLineNo">321</span> // shutdown and ensure that Reader gets right length also.<a name="line.321"></a>
+<span class="sourceLineNo">322</span> wal.shutdown();<a name="line.322"></a>
+<span class="sourceLineNo">323</span> reader = wals.createReader(fs, walPath);<a name="line.323"></a>
+<span class="sourceLineNo">324</span> count = 0;<a name="line.324"></a>
+<span class="sourceLineNo">325</span> while((entry = reader.next(entry)) != null) count++;<a name="line.325"></a>
+<span class="sourceLineNo">326</span> assertEquals(total * 3, count);<a name="line.326"></a>
+<span class="sourceLineNo">327</span> reader.close();<a name="line.327"></a>
+<span class="sourceLineNo">328</span> } finally {<a name="line.328"></a>
+<span class="sourceLineNo">329</span> if (reader != null) reader.close();<a na
<TRUNCATED>
[17/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.html b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.html
index fcb6bb2..6e49a29 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/WALSplitter.html
@@ -256,43 +256,43 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#regionMaxSeqIdInStores">regionMaxSeqIdInStores</a></span></code> </td>
</tr>
<tr class="rowColor">
-<td class="colFirst"><code>protected org.apache.hadoop.fs.Path</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#rootDir">rootDir</a></span></code> </td>
-</tr>
-<tr class="altColor">
<td class="colFirst"><code>private static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#SEQUENCE_ID_FILE_SUFFIX">SEQUENCE_ID_FILE_SUFFIX</a></span></code> </td>
</tr>
-<tr class="rowColor">
+<tr class="altColor">
<td class="colFirst"><code>private static int</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#SEQUENCE_ID_FILE_SUFFIX_LENGTH">SEQUENCE_ID_FILE_SUFFIX_LENGTH</a></span></code> </td>
</tr>
-<tr class="altColor">
+<tr class="rowColor">
<td class="colFirst"><code>protected <a href="../../../../../org/apache/hadoop/hbase/regionserver/LastSequenceId.html" title="interface in org.apache.hadoop.hbase.regionserver">LastSequenceId</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#sequenceIdChecker">sequenceIdChecker</a></span></code> </td>
</tr>
-<tr class="rowColor">
+<tr class="altColor">
<td class="colFirst"><code>static boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#SPLIT_SKIP_ERRORS_DEFAULT">SPLIT_SKIP_ERRORS_DEFAULT</a></span></code>
<div class="block">By default we retry errors in splitting, rather than skipping.</div>
</td>
</tr>
-<tr class="altColor">
+<tr class="rowColor">
<td class="colFirst"><code>static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#SPLIT_WRITER_CREATION_BOUNDED">SPLIT_WRITER_CREATION_BOUNDED</a></span></code> </td>
</tr>
-<tr class="rowColor">
+<tr class="altColor">
<td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/coordination/SplitLogWorkerCoordination.html" title="interface in org.apache.hadoop.hbase.coordination">SplitLogWorkerCoordination</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogWorkerCoordination">splitLogWorkerCoordination</a></span></code> </td>
</tr>
-<tr class="altColor">
+<tr class="rowColor">
<td class="colFirst"><code>private boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitWriterCreationBounded">splitWriterCreationBounded</a></span></code> </td>
</tr>
-<tr class="rowColor">
+<tr class="altColor">
<td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/monitoring/MonitoredTask.html" title="interface in org.apache.hadoop.hbase.monitoring">MonitoredTask</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#status">status</a></span></code> </td>
</tr>
+<tr class="rowColor">
+<td class="colFirst"><code>protected org.apache.hadoop.fs.Path</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#walDir">walDir</a></span></code> </td>
+</tr>
<tr class="altColor">
<td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/wal/WALFactory.html" title="class in org.apache.hadoop.hbase.wal">WALFactory</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#walFactory">walFactory</a></span></code> </td>
@@ -314,7 +314,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#WALSplitter-org.apache.hadoop.hbase.wal.WALFactory-org.apache.hadoop.conf.Configuration-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-">WALSplitter</a></span>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALFactory.html" title="class in org.apache.hadoop.hbase.wal">WALFactory</a> factory,
org.apache.hadoop.conf.Configuration conf,
- org.apache.hadoop.fs.Path rootDir,
+ org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileSystem fs,
<a href="../../../../../org/apache/hadoop/hbase/regionserver/LastSequenceId.html" title="interface in org.apache.hadoop.hbase.regionserver">LastSequenceId</a> idChecker,
<a href="../../../../../org/apache/hadoop/hbase/coordination/SplitLogWorkerCoordination.html" title="interface in org.apache.hadoop.hbase.coordination">SplitLogWorkerCoordination</a> splitLogWorkerCoordination)</code> </td>
@@ -431,10 +431,9 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr id="i14" class="altColor">
<td class="colFirst"><code>(package private) static org.apache.hadoop.fs.Path</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#getRegionSplitEditsPath-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-java.lang.String-">getRegionSplitEditsPath</a></span>(org.apache.hadoop.fs.FileSystem fs,
- <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry,
- org.apache.hadoop.fs.Path rootDir,
- <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileNameBeingSplit)</code>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#getRegionSplitEditsPath-org.apache.hadoop.hbase.wal.WAL.Entry-java.lang.String-org.apache.hadoop.conf.Configuration-">getRegionSplitEditsPath</a></span>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry,
+ <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileNameBeingSplit,
+ org.apache.hadoop.conf.Configuration conf)</code>
<div class="block">Path to a file under RECOVERED_EDITS_DIR directory of the region found in
<code>logEntry</code> named for the sequenceid in the passed
<code>logEntry</code>: e.g.</div>
@@ -488,7 +487,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr id="i22" class="altColor">
<td class="colFirst"><code>static boolean</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path rootDir,
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
@@ -552,13 +551,13 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</dl>
</li>
</ul>
-<a name="rootDir">
+<a name="walDir">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
-<h4>rootDir</h4>
-<pre>protected final org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.118">rootDir</a></pre>
+<h4>walDir</h4>
+<pre>protected final org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.118">walDir</a></pre>
</li>
</ul>
<a name="fs">
@@ -688,7 +687,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>EDITFILES_NAME_PATTERN</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html?is-external=true" title="class or interface in java.util.regex">Pattern</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.532">EDITFILES_NAME_PATTERN</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html?is-external=true" title="class or interface in java.util.regex">Pattern</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.533">EDITFILES_NAME_PATTERN</a></pre>
</li>
</ul>
<a name="RECOVERED_LOG_TMPFILE_SUFFIX">
@@ -697,7 +696,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>RECOVERED_LOG_TMPFILE_SUFFIX</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.533">RECOVERED_LOG_TMPFILE_SUFFIX</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.534">RECOVERED_LOG_TMPFILE_SUFFIX</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.wal.WALSplitter.RECOVERED_LOG_TMPFILE_SUFFIX">Constant Field Values</a></dd>
@@ -710,7 +709,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>SEQUENCE_ID_FILE_SUFFIX</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.612">SEQUENCE_ID_FILE_SUFFIX</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.613">SEQUENCE_ID_FILE_SUFFIX</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.wal.WALSplitter.SEQUENCE_ID_FILE_SUFFIX">Constant Field Values</a></dd>
@@ -723,7 +722,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>OLD_SEQUENCE_ID_FILE_SUFFIX</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.613">OLD_SEQUENCE_ID_FILE_SUFFIX</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.614">OLD_SEQUENCE_ID_FILE_SUFFIX</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.wal.WALSplitter.OLD_SEQUENCE_ID_FILE_SUFFIX">Constant Field Values</a></dd>
@@ -736,7 +735,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>SEQUENCE_ID_FILE_SUFFIX_LENGTH</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.614">SEQUENCE_ID_FILE_SUFFIX_LENGTH</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.615">SEQUENCE_ID_FILE_SUFFIX_LENGTH</a></pre>
</li>
</ul>
</li>
@@ -755,7 +754,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<h4>WALSplitter</h4>
<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.151">WALSplitter</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WALFactory.html" title="class in org.apache.hadoop.hbase.wal">WALFactory</a> factory,
org.apache.hadoop.conf.Configuration conf,
- org.apache.hadoop.fs.Path rootDir,
+ org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileSystem fs,
<a href="../../../../../org/apache/hadoop/hbase/regionserver/LastSequenceId.html" title="interface in org.apache.hadoop.hbase.regionserver">LastSequenceId</a> idChecker,
<a href="../../../../../org/apache/hadoop/hbase/coordination/SplitLogWorkerCoordination.html" title="interface in org.apache.hadoop.hbase.coordination">SplitLogWorkerCoordination</a> splitLogWorkerCoordination)</pre>
@@ -775,7 +774,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>splitLogFile</h4>
-<pre>public static boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.189">splitLogFile</a>(org.apache.hadoop.fs.Path rootDir,
+<pre>public static boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.189">splitLogFile</a>(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
@@ -902,16 +901,15 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</dl>
</li>
</ul>
-<a name="getRegionSplitEditsPath-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-java.lang.String-">
+<a name="getRegionSplitEditsPath-org.apache.hadoop.hbase.wal.WAL.Entry-java.lang.String-org.apache.hadoop.conf.Configuration-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getRegionSplitEditsPath</h4>
-<pre>static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.469">getRegionSplitEditsPath</a>(org.apache.hadoop.fs.FileSystem fs,
- <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry,
- org.apache.hadoop.fs.Path rootDir,
- <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileNameBeingSplit)
+<pre>static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.468">getRegionSplitEditsPath</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry,
+ <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileNameBeingSplit,
+ org.apache.hadoop.conf.Configuration conf)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Path to a file under RECOVERED_EDITS_DIR directory of the region found in
<code>logEntry</code> named for the sequenceid in the passed
@@ -920,10 +918,9 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
creating it if necessary.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
-<dd><code>fs</code> - </dd>
<dd><code>logEntry</code> - </dd>
-<dd><code>rootDir</code> - HBase root dir.</dd>
<dd><code>fileNameBeingSplit</code> - the file being split currently. Used to generate tmp file name.</dd>
+<dd><code>conf</code> - </dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>Path to file into which to dump split log edits.</dd>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -937,7 +934,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getTmpRecoveredEditsFileName</h4>
-<pre>private static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.509">getTmpRecoveredEditsFileName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileName)</pre>
+<pre>private static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.510">getTmpRecoveredEditsFileName</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileName)</pre>
</li>
</ul>
<a name="getCompletedRecoveredEditsFilePath-org.apache.hadoop.fs.Path-long-">
@@ -946,7 +943,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getCompletedRecoveredEditsFilePath</h4>
-<pre>private static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.521">getCompletedRecoveredEditsFilePath</a>(org.apache.hadoop.fs.Path srcPath,
+<pre>private static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.522">getCompletedRecoveredEditsFilePath</a>(org.apache.hadoop.fs.Path srcPath,
long maximumEditLogSeqNum)</pre>
<div class="block">Get the completed recovered edits file path, renaming it to be by last edit
in the file from its first edit. Then we could use the name to skip
@@ -966,7 +963,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>formatRecoveredEditsFileName</h4>
-<pre>static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.528">formatRecoveredEditsFileName</a>(long seqid)</pre>
+<pre>static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.529">formatRecoveredEditsFileName</a>(long seqid)</pre>
</li>
</ul>
<a name="getRegionDirRecoveredEditsDir-org.apache.hadoop.fs.Path-">
@@ -975,7 +972,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getRegionDirRecoveredEditsDir</h4>
-<pre>public static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.541">getRegionDirRecoveredEditsDir</a>(org.apache.hadoop.fs.Path regiondir)</pre>
+<pre>public static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.542">getRegionDirRecoveredEditsDir</a>(org.apache.hadoop.fs.Path regiondir)</pre>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>regiondir</code> - This regions directory in the filesystem.</dd>
@@ -991,7 +988,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getSplitEditFilesSorted</h4>
-<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/util/NavigableSet.html?is-external=true" title="class or interface in java.util">NavigableSet</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.554">getSplitEditFilesSorted</a>(org.apache.hadoop.fs.FileSystem fs,
+<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/util/NavigableSet.html?is-external=true" title="class or interface in java.util">NavigableSet</a><org.apache.hadoop.fs.Path> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.555">getSplitEditFilesSorted</a>(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regiondir)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Returns sorted set of edit files made by splitter, excluding files
@@ -1013,7 +1010,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>moveAsideBadEditsFile</h4>
-<pre>public static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.602">moveAsideBadEditsFile</a>(org.apache.hadoop.fs.FileSystem fs,
+<pre>public static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.603">moveAsideBadEditsFile</a>(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path edits)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Move aside a bad edits file.</div>
@@ -1034,7 +1031,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isSequenceIdFile</h4>
-<pre>public static boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.620">isSequenceIdFile</a>(org.apache.hadoop.fs.Path file)</pre>
+<pre>public static boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.621">isSequenceIdFile</a>(org.apache.hadoop.fs.Path file)</pre>
<div class="block">Is the given file a region open sequence id file.</div>
</li>
</ul>
@@ -1044,7 +1041,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getSequenceIdFiles</h4>
-<pre>private static org.apache.hadoop.fs.FileStatus[] <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.625">getSequenceIdFiles</a>(org.apache.hadoop.fs.FileSystem fs,
+<pre>private static org.apache.hadoop.fs.FileStatus[] <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.626">getSequenceIdFiles</a>(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regionDir)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -1059,7 +1056,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getMaxSequenceId</h4>
-<pre>private static long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.637">getMaxSequenceId</a>(org.apache.hadoop.fs.FileStatus[] files)</pre>
+<pre>private static long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.638">getMaxSequenceId</a>(org.apache.hadoop.fs.FileStatus[] files)</pre>
</li>
</ul>
<a name="getMaxRegionSequenceId-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.fs.Path-">
@@ -1068,7 +1065,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getMaxRegionSequenceId</h4>
-<pre>public static long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.654">getMaxRegionSequenceId</a>(org.apache.hadoop.fs.FileSystem fs,
+<pre>public static long <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.655">getMaxRegionSequenceId</a>(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regionDir)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Get the max sequence id which is stored in the region directory. -1 if none.</div>
@@ -1084,7 +1081,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>writeRegionSequenceIdFile</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.661">writeRegionSequenceIdFile</a>(org.apache.hadoop.fs.FileSystem fs,
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.662">writeRegionSequenceIdFile</a>(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path regionDir,
long newMaxSeqId)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -1101,7 +1098,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getReader</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal">WAL.Reader</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.699">getReader</a>(org.apache.hadoop.fs.FileStatus file,
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal">WAL.Reader</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.700">getReader</a>(org.apache.hadoop.fs.FileStatus file,
boolean skipErrors,
<a href="../../../../../org/apache/hadoop/hbase/util/CancelableProgressable.html" title="interface in org.apache.hadoop.hbase.util">CancelableProgressable</a> reporter)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
@@ -1124,7 +1121,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getNextLogLine</h4>
-<pre>private static <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.746">getNextLogLine</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal">WAL.Reader</a> in,
+<pre>private static <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.747">getNextLogLine</a>(<a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal">WAL.Reader</a> in,
org.apache.hadoop.fs.Path path,
boolean skipErrors)
throws <a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.CorruptedLogFileException.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.CorruptedLogFileException</a>,
@@ -1142,7 +1139,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>createWriter</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.778">createWriter</a>(org.apache.hadoop.fs.Path logfile)
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal">WALProvider.Writer</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.779">createWriter</a>(org.apache.hadoop.fs.Path logfile)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Create a new <a href="../../../../../org/apache/hadoop/hbase/wal/WALProvider.Writer.html" title="interface in org.apache.hadoop.hbase.wal"><code>WALProvider.Writer</code></a> for writing log splits.</div>
<dl>
@@ -1159,7 +1156,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getReader</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal">WAL.Reader</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.787">getReader</a>(org.apache.hadoop.fs.Path curLogFile,
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal">WAL.Reader</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.788">getReader</a>(org.apache.hadoop.fs.Path curLogFile,
<a href="../../../../../org/apache/hadoop/hbase/util/CancelableProgressable.html" title="interface in org.apache.hadoop.hbase.util">CancelableProgressable</a> reporter)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Create a new <a href="../../../../../org/apache/hadoop/hbase/wal/WAL.Reader.html" title="interface in org.apache.hadoop.hbase.wal"><code>WAL.Reader</code></a> for reading logs to split.</div>
@@ -1177,7 +1174,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getNumOpenWriters</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.794">getNumOpenWriters</a>()</pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.795">getNumOpenWriters</a>()</pre>
<div class="block">Get current open writers</div>
</li>
</ul>
@@ -1187,7 +1184,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>getMutationsFromWALEntry</h4>
-<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.MutationReplay</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1818">getMutationsFromWALEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry entry,
+<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALSplitter.MutationReplay.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.MutationReplay</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/WALSplitter.html#line.1823">getMutationsFromWALEntry</a>(org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.WALEntry entry,
<a href="../../../../../org/apache/hadoop/hbase/CellScanner.html" title="interface in org.apache.hadoop.hbase">CellScanner</a> cells,
<a href="../../../../../org/apache/hadoop/hbase/util/Pair.html" title="class in org.apache.hadoop.hbase.util">Pair</a><<a href="../../../../../org/apache/hadoop/hbase/wal/WALKey.html" title="interface in org.apache.hadoop.hbase.wal">WALKey</a>,<a href="../../../../../org/apache/hadoop/hbase/wal/WALEdit.html" title="class in org.apache.hadoop.hbase.wal">WALEdit</a>> logEntry,
<a href="../../../../../org/apache/hadoop/hbase/client/Durability.html" title="enum in org.apache.hadoop.hbase.client">Durability</a> durability)
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/class-use/WAL.Entry.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/class-use/WAL.Entry.html b/devapidocs/org/apache/hadoop/hbase/wal/class-use/WAL.Entry.html
index 4edf533..d2bf8ee 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/class-use/WAL.Entry.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/class-use/WAL.Entry.html
@@ -666,9 +666,8 @@ Input/OutputFormats, a table indexing MapReduce job, and utility methods.</div>
</tr>
<tr class="altColor">
<td class="colFirst"><code>(package private) <a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a></code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.LogRecoveredEditsOutputSink.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-">createWAP</a></span>(byte[] region,
- <a href="../../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry,
- org.apache.hadoop.fs.Path rootdir)</code> </td>
+<td class="colLast"><span class="typeNameLabel">WALSplitter.LogRecoveredEditsOutputSink.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-">createWAP</a></span>(byte[] region,
+ <a href="../../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>(package private) void</code></td>
@@ -676,10 +675,9 @@ Input/OutputFormats, a table indexing MapReduce job, and utility methods.</div>
</tr>
<tr class="altColor">
<td class="colFirst"><code>(package private) static org.apache.hadoop.fs.Path</code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#getRegionSplitEditsPath-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-java.lang.String-">getRegionSplitEditsPath</a></span>(org.apache.hadoop.fs.FileSystem fs,
- <a href="../../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry,
- org.apache.hadoop.fs.Path rootDir,
- <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileNameBeingSplit)</code>
+<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#getRegionSplitEditsPath-org.apache.hadoop.hbase.wal.WAL.Entry-java.lang.String-org.apache.hadoop.conf.Configuration-">getRegionSplitEditsPath</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> logEntry,
+ <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> fileNameBeingSplit,
+ org.apache.hadoop.conf.Configuration conf)</code>
<div class="block">Path to a file under RECOVERED_EDITS_DIR directory of the region found in
<code>logEntry</code> named for the sequenceid in the passed
<code>logEntry</code>: e.g.</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALFactory.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALFactory.html b/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALFactory.html
index 23d714f..e782d4d 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALFactory.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALFactory.html
@@ -223,7 +223,7 @@
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static boolean</code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path rootDir,
+<td class="colLast"><span class="typeNameLabel">WALSplitter.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#splitLogFile-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileStatus-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.conf.Configuration-org.apache.hadoop.hbase.util.CancelableProgressable-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-org.apache.hadoop.hbase.wal.WALFactory-">splitLogFile</a></span>(org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileStatus logfile,
org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.conf.Configuration conf,
@@ -245,7 +245,7 @@
<tr class="altColor">
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.html#WALSplitter-org.apache.hadoop.hbase.wal.WALFactory-org.apache.hadoop.conf.Configuration-org.apache.hadoop.fs.Path-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.regionserver.LastSequenceId-org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination-">WALSplitter</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/wal/WALFactory.html" title="class in org.apache.hadoop.hbase.wal">WALFactory</a> factory,
org.apache.hadoop.conf.Configuration conf,
- org.apache.hadoop.fs.Path rootDir,
+ org.apache.hadoop.fs.Path walDir,
org.apache.hadoop.fs.FileSystem fs,
<a href="../../../../../../org/apache/hadoop/hbase/regionserver/LastSequenceId.html" title="interface in org.apache.hadoop.hbase.regionserver">LastSequenceId</a> idChecker,
<a href="../../../../../../org/apache/hadoop/hbase/coordination/SplitLogWorkerCoordination.html" title="interface in org.apache.hadoop.hbase.coordination">SplitLogWorkerCoordination</a> splitLogWorkerCoordination)</code> </td>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALSplitter.WriterAndPath.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALSplitter.WriterAndPath.html b/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALSplitter.WriterAndPath.html
index 29c3617..a578e6a 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALSplitter.WriterAndPath.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/class-use/WALSplitter.WriterAndPath.html
@@ -109,9 +109,8 @@
</tr>
<tr class="rowColor">
<td class="colFirst"><code>(package private) <a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a></code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.LogRecoveredEditsOutputSink.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-org.apache.hadoop.fs.Path-">createWAP</a></span>(byte[] region,
- <a href="../../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry,
- org.apache.hadoop.fs.Path rootdir)</code> </td>
+<td class="colLast"><span class="typeNameLabel">WALSplitter.LogRecoveredEditsOutputSink.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#createWAP-byte:A-org.apache.hadoop.hbase.wal.WAL.Entry-">createWAP</a></span>(byte[] region,
+ <a href="../../../../../../org/apache/hadoop/hbase/wal/WAL.Entry.html" title="class in org.apache.hadoop.hbase.wal">WAL.Entry</a> entry)</code> </td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>(package private) <a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a></code></td>
@@ -137,7 +136,8 @@
</tr>
<tr class="rowColor">
<td class="colFirst"><code>private void</code></td>
-<td class="colLast"><span class="typeNameLabel">WALSplitter.LogRecoveredEditsOutputSink.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#deleteOneWithFewerEntries-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">deleteOneWithFewerEntries</a></span>(<a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
+<td class="colLast"><span class="typeNameLabel">WALSplitter.LogRecoveredEditsOutputSink.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.LogRecoveredEditsOutputSink.html#deleteOneWithFewerEntries-org.apache.hadoop.fs.FileSystem-org.apache.hadoop.hbase.wal.WALSplitter.WriterAndPath-org.apache.hadoop.fs.Path-">deleteOneWithFewerEntries</a></span>(org.apache.hadoop.fs.FileSystem rootFs,
+ <a href="../../../../../../org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html" title="class in org.apache.hadoop.hbase.wal">WALSplitter.WriterAndPath</a> wap,
org.apache.hadoop.fs.Path dst)</code> </td>
</tr>
</tbody>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
index b696c50..a092a6a 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
@@ -16,11 +16,11 @@
<span class="sourceLineNo">008</span>@InterfaceAudience.Private<a name="line.8"></a>
<span class="sourceLineNo">009</span>public class Version {<a name="line.9"></a>
<span class="sourceLineNo">010</span> public static final String version = "3.0.0-SNAPSHOT";<a name="line.10"></a>
-<span class="sourceLineNo">011</span> public static final String revision = "0e43abc78a5a55c84a7572125d498a950dabfb57";<a name="line.11"></a>
+<span class="sourceLineNo">011</span> public static final String revision = "ac5bb8155b618194fe9cf1131f0e72c99b7b534c";<a name="line.11"></a>
<span class="sourceLineNo">012</span> public static final String user = "jenkins";<a name="line.12"></a>
-<span class="sourceLineNo">013</span> public static final String date = "Fri Jun 15 21:36:50 UTC 2018";<a name="line.13"></a>
+<span class="sourceLineNo">013</span> public static final String date = "Sat Jun 16 21:34:54 UTC 2018";<a name="line.13"></a>
<span class="sourceLineNo">014</span> public static final String url = "git://jenkins-websites1.apache.org/home/jenkins/jenkins-slave/workspace/hbase_generate_website/hbase";<a name="line.14"></a>
-<span class="sourceLineNo">015</span> public static final String srcChecksum = "2e5366d5b65adc25ba60d4644effc4f6";<a name="line.15"></a>
+<span class="sourceLineNo">015</span> public static final String srcChecksum = "72ef87fb3fa82a7c510def4b3609400b";<a name="line.15"></a>
<span class="sourceLineNo">016</span>}<a name="line.16"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.Writer.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.Writer.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.Writer.html
index 7766850..74d8640 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.Writer.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.Writer.html
@@ -81,41 +81,42 @@
<span class="sourceLineNo">073</span> Writer writer = null;<a name="line.73"></a>
<span class="sourceLineNo">074</span> try {<a name="line.74"></a>
<span class="sourceLineNo">075</span> writer = logWriterClass.getDeclaredConstructor().newInstance();<a name="line.75"></a>
-<span class="sourceLineNo">076</span> writer.init(fs, path, conf, overwritable, blocksize);<a name="line.76"></a>
-<span class="sourceLineNo">077</span> return writer;<a name="line.77"></a>
-<span class="sourceLineNo">078</span> } catch (Exception e) { <a name="line.78"></a>
-<span class="sourceLineNo">079</span> if (e instanceof CommonFSUtils.StreamLacksCapabilityException) {<a name="line.79"></a>
-<span class="sourceLineNo">080</span> LOG.error("The RegionServer write ahead log provider for FileSystem implementations " +<a name="line.80"></a>
-<span class="sourceLineNo">081</span> "relies on the ability to call " + e.getMessage() + " for proper operation during " +<a name="line.81"></a>
-<span class="sourceLineNo">082</span> "component failures, but the current FileSystem does not support doing so. Please " +<a name="line.82"></a>
-<span class="sourceLineNo">083</span> "check the config value of '" + CommonFSUtils.HBASE_WAL_DIR + "' and ensure " +<a name="line.83"></a>
-<span class="sourceLineNo">084</span> "it points to a FileSystem mount that has suitable capabilities for output streams.");<a name="line.84"></a>
-<span class="sourceLineNo">085</span> } else {<a name="line.85"></a>
-<span class="sourceLineNo">086</span> LOG.debug("Error instantiating log writer.", e);<a name="line.86"></a>
-<span class="sourceLineNo">087</span> }<a name="line.87"></a>
-<span class="sourceLineNo">088</span> if (writer != null) {<a name="line.88"></a>
-<span class="sourceLineNo">089</span> try{<a name="line.89"></a>
-<span class="sourceLineNo">090</span> writer.close();<a name="line.90"></a>
-<span class="sourceLineNo">091</span> } catch(IOException ee){<a name="line.91"></a>
-<span class="sourceLineNo">092</span> LOG.error("cannot close log writer", ee);<a name="line.92"></a>
-<span class="sourceLineNo">093</span> }<a name="line.93"></a>
-<span class="sourceLineNo">094</span> }<a name="line.94"></a>
-<span class="sourceLineNo">095</span> throw new IOException("cannot get log writer", e);<a name="line.95"></a>
-<span class="sourceLineNo">096</span> }<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span><a name="line.98"></a>
-<span class="sourceLineNo">099</span> @Override<a name="line.99"></a>
-<span class="sourceLineNo">100</span> protected FSHLog createWAL() throws IOException {<a name="line.100"></a>
-<span class="sourceLineNo">101</span> return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),<a name="line.101"></a>
-<span class="sourceLineNo">102</span> getWALDirectoryName(factory.factoryId),<a name="line.102"></a>
-<span class="sourceLineNo">103</span> getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix,<a name="line.103"></a>
-<span class="sourceLineNo">104</span> META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);<a name="line.104"></a>
-<span class="sourceLineNo">105</span> }<a name="line.105"></a>
-<span class="sourceLineNo">106</span><a name="line.106"></a>
-<span class="sourceLineNo">107</span> @Override<a name="line.107"></a>
-<span class="sourceLineNo">108</span> protected void doInit(Configuration conf) throws IOException {<a name="line.108"></a>
-<span class="sourceLineNo">109</span> }<a name="line.109"></a>
-<span class="sourceLineNo">110</span>}<a name="line.110"></a>
+<span class="sourceLineNo">076</span> FileSystem rootFs = FileSystem.get(path.toUri(), conf);<a name="line.76"></a>
+<span class="sourceLineNo">077</span> writer.init(rootFs, path, conf, overwritable, blocksize);<a name="line.77"></a>
+<span class="sourceLineNo">078</span> return writer;<a name="line.78"></a>
+<span class="sourceLineNo">079</span> } catch (Exception e) { <a name="line.79"></a>
+<span class="sourceLineNo">080</span> if (e instanceof CommonFSUtils.StreamLacksCapabilityException) {<a name="line.80"></a>
+<span class="sourceLineNo">081</span> LOG.error("The RegionServer write ahead log provider for FileSystem implementations " +<a name="line.81"></a>
+<span class="sourceLineNo">082</span> "relies on the ability to call " + e.getMessage() + " for proper operation during " +<a name="line.82"></a>
+<span class="sourceLineNo">083</span> "component failures, but the current FileSystem does not support doing so. Please " +<a name="line.83"></a>
+<span class="sourceLineNo">084</span> "check the config value of '" + CommonFSUtils.HBASE_WAL_DIR + "' and ensure " +<a name="line.84"></a>
+<span class="sourceLineNo">085</span> "it points to a FileSystem mount that has suitable capabilities for output streams.");<a name="line.85"></a>
+<span class="sourceLineNo">086</span> } else {<a name="line.86"></a>
+<span class="sourceLineNo">087</span> LOG.debug("Error instantiating log writer.", e);<a name="line.87"></a>
+<span class="sourceLineNo">088</span> }<a name="line.88"></a>
+<span class="sourceLineNo">089</span> if (writer != null) {<a name="line.89"></a>
+<span class="sourceLineNo">090</span> try{<a name="line.90"></a>
+<span class="sourceLineNo">091</span> writer.close();<a name="line.91"></a>
+<span class="sourceLineNo">092</span> } catch(IOException ee){<a name="line.92"></a>
+<span class="sourceLineNo">093</span> LOG.error("cannot close log writer", ee);<a name="line.93"></a>
+<span class="sourceLineNo">094</span> }<a name="line.94"></a>
+<span class="sourceLineNo">095</span> }<a name="line.95"></a>
+<span class="sourceLineNo">096</span> throw new IOException("cannot get log writer", e);<a name="line.96"></a>
+<span class="sourceLineNo">097</span> }<a name="line.97"></a>
+<span class="sourceLineNo">098</span> }<a name="line.98"></a>
+<span class="sourceLineNo">099</span><a name="line.99"></a>
+<span class="sourceLineNo">100</span> @Override<a name="line.100"></a>
+<span class="sourceLineNo">101</span> protected FSHLog createWAL() throws IOException {<a name="line.101"></a>
+<span class="sourceLineNo">102</span> return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),<a name="line.102"></a>
+<span class="sourceLineNo">103</span> getWALDirectoryName(factory.factoryId),<a name="line.103"></a>
+<span class="sourceLineNo">104</span> getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix,<a name="line.104"></a>
+<span class="sourceLineNo">105</span> META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);<a name="line.105"></a>
+<span class="sourceLineNo">106</span> }<a name="line.106"></a>
+<span class="sourceLineNo">107</span><a name="line.107"></a>
+<span class="sourceLineNo">108</span> @Override<a name="line.108"></a>
+<span class="sourceLineNo">109</span> protected void doInit(Configuration conf) throws IOException {<a name="line.109"></a>
+<span class="sourceLineNo">110</span> }<a name="line.110"></a>
+<span class="sourceLineNo">111</span>}<a name="line.111"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html
index 7766850..74d8640 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/FSHLogProvider.html
@@ -81,41 +81,42 @@
<span class="sourceLineNo">073</span> Writer writer = null;<a name="line.73"></a>
<span class="sourceLineNo">074</span> try {<a name="line.74"></a>
<span class="sourceLineNo">075</span> writer = logWriterClass.getDeclaredConstructor().newInstance();<a name="line.75"></a>
-<span class="sourceLineNo">076</span> writer.init(fs, path, conf, overwritable, blocksize);<a name="line.76"></a>
-<span class="sourceLineNo">077</span> return writer;<a name="line.77"></a>
-<span class="sourceLineNo">078</span> } catch (Exception e) { <a name="line.78"></a>
-<span class="sourceLineNo">079</span> if (e instanceof CommonFSUtils.StreamLacksCapabilityException) {<a name="line.79"></a>
-<span class="sourceLineNo">080</span> LOG.error("The RegionServer write ahead log provider for FileSystem implementations " +<a name="line.80"></a>
-<span class="sourceLineNo">081</span> "relies on the ability to call " + e.getMessage() + " for proper operation during " +<a name="line.81"></a>
-<span class="sourceLineNo">082</span> "component failures, but the current FileSystem does not support doing so. Please " +<a name="line.82"></a>
-<span class="sourceLineNo">083</span> "check the config value of '" + CommonFSUtils.HBASE_WAL_DIR + "' and ensure " +<a name="line.83"></a>
-<span class="sourceLineNo">084</span> "it points to a FileSystem mount that has suitable capabilities for output streams.");<a name="line.84"></a>
-<span class="sourceLineNo">085</span> } else {<a name="line.85"></a>
-<span class="sourceLineNo">086</span> LOG.debug("Error instantiating log writer.", e);<a name="line.86"></a>
-<span class="sourceLineNo">087</span> }<a name="line.87"></a>
-<span class="sourceLineNo">088</span> if (writer != null) {<a name="line.88"></a>
-<span class="sourceLineNo">089</span> try{<a name="line.89"></a>
-<span class="sourceLineNo">090</span> writer.close();<a name="line.90"></a>
-<span class="sourceLineNo">091</span> } catch(IOException ee){<a name="line.91"></a>
-<span class="sourceLineNo">092</span> LOG.error("cannot close log writer", ee);<a name="line.92"></a>
-<span class="sourceLineNo">093</span> }<a name="line.93"></a>
-<span class="sourceLineNo">094</span> }<a name="line.94"></a>
-<span class="sourceLineNo">095</span> throw new IOException("cannot get log writer", e);<a name="line.95"></a>
-<span class="sourceLineNo">096</span> }<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span><a name="line.98"></a>
-<span class="sourceLineNo">099</span> @Override<a name="line.99"></a>
-<span class="sourceLineNo">100</span> protected FSHLog createWAL() throws IOException {<a name="line.100"></a>
-<span class="sourceLineNo">101</span> return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),<a name="line.101"></a>
-<span class="sourceLineNo">102</span> getWALDirectoryName(factory.factoryId),<a name="line.102"></a>
-<span class="sourceLineNo">103</span> getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix,<a name="line.103"></a>
-<span class="sourceLineNo">104</span> META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);<a name="line.104"></a>
-<span class="sourceLineNo">105</span> }<a name="line.105"></a>
-<span class="sourceLineNo">106</span><a name="line.106"></a>
-<span class="sourceLineNo">107</span> @Override<a name="line.107"></a>
-<span class="sourceLineNo">108</span> protected void doInit(Configuration conf) throws IOException {<a name="line.108"></a>
-<span class="sourceLineNo">109</span> }<a name="line.109"></a>
-<span class="sourceLineNo">110</span>}<a name="line.110"></a>
+<span class="sourceLineNo">076</span> FileSystem rootFs = FileSystem.get(path.toUri(), conf);<a name="line.76"></a>
+<span class="sourceLineNo">077</span> writer.init(rootFs, path, conf, overwritable, blocksize);<a name="line.77"></a>
+<span class="sourceLineNo">078</span> return writer;<a name="line.78"></a>
+<span class="sourceLineNo">079</span> } catch (Exception e) { <a name="line.79"></a>
+<span class="sourceLineNo">080</span> if (e instanceof CommonFSUtils.StreamLacksCapabilityException) {<a name="line.80"></a>
+<span class="sourceLineNo">081</span> LOG.error("The RegionServer write ahead log provider for FileSystem implementations " +<a name="line.81"></a>
+<span class="sourceLineNo">082</span> "relies on the ability to call " + e.getMessage() + " for proper operation during " +<a name="line.82"></a>
+<span class="sourceLineNo">083</span> "component failures, but the current FileSystem does not support doing so. Please " +<a name="line.83"></a>
+<span class="sourceLineNo">084</span> "check the config value of '" + CommonFSUtils.HBASE_WAL_DIR + "' and ensure " +<a name="line.84"></a>
+<span class="sourceLineNo">085</span> "it points to a FileSystem mount that has suitable capabilities for output streams.");<a name="line.85"></a>
+<span class="sourceLineNo">086</span> } else {<a name="line.86"></a>
+<span class="sourceLineNo">087</span> LOG.debug("Error instantiating log writer.", e);<a name="line.87"></a>
+<span class="sourceLineNo">088</span> }<a name="line.88"></a>
+<span class="sourceLineNo">089</span> if (writer != null) {<a name="line.89"></a>
+<span class="sourceLineNo">090</span> try{<a name="line.90"></a>
+<span class="sourceLineNo">091</span> writer.close();<a name="line.91"></a>
+<span class="sourceLineNo">092</span> } catch(IOException ee){<a name="line.92"></a>
+<span class="sourceLineNo">093</span> LOG.error("cannot close log writer", ee);<a name="line.93"></a>
+<span class="sourceLineNo">094</span> }<a name="line.94"></a>
+<span class="sourceLineNo">095</span> }<a name="line.95"></a>
+<span class="sourceLineNo">096</span> throw new IOException("cannot get log writer", e);<a name="line.96"></a>
+<span class="sourceLineNo">097</span> }<a name="line.97"></a>
+<span class="sourceLineNo">098</span> }<a name="line.98"></a>
+<span class="sourceLineNo">099</span><a name="line.99"></a>
+<span class="sourceLineNo">100</span> @Override<a name="line.100"></a>
+<span class="sourceLineNo">101</span> protected FSHLog createWAL() throws IOException {<a name="line.101"></a>
+<span class="sourceLineNo">102</span> return new FSHLog(CommonFSUtils.getWALFileSystem(conf), CommonFSUtils.getWALRootDir(conf),<a name="line.102"></a>
+<span class="sourceLineNo">103</span> getWALDirectoryName(factory.factoryId),<a name="line.103"></a>
+<span class="sourceLineNo">104</span> getWALArchiveDirectoryName(conf, factory.factoryId), conf, listeners, true, logPrefix,<a name="line.104"></a>
+<span class="sourceLineNo">105</span> META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);<a name="line.105"></a>
+<span class="sourceLineNo">106</span> }<a name="line.106"></a>
+<span class="sourceLineNo">107</span><a name="line.107"></a>
+<span class="sourceLineNo">108</span> @Override<a name="line.108"></a>
+<span class="sourceLineNo">109</span> protected void doInit(Configuration conf) throws IOException {<a name="line.109"></a>
+<span class="sourceLineNo">110</span> }<a name="line.110"></a>
+<span class="sourceLineNo">111</span>}<a name="line.111"></a>
[03/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
index db8cddf..345e682 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
@@ -65,621 +65,623 @@
<span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.testclassification.MediumTests;<a name="line.57"></a>
<span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.testclassification.RegionServerTests;<a name="line.58"></a>
<span class="sourceLineNo">059</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.59"></a>
-<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.60"></a>
-<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.61"></a>
-<span class="sourceLineNo">062</span>import org.apache.hadoop.hdfs.DistributedFileSystem;<a name="line.62"></a>
-<span class="sourceLineNo">063</span>import org.apache.hadoop.hdfs.MiniDFSCluster;<a name="line.63"></a>
-<span class="sourceLineNo">064</span>import org.apache.hadoop.hdfs.protocol.HdfsConstants;<a name="line.64"></a>
-<span class="sourceLineNo">065</span>import org.junit.After;<a name="line.65"></a>
-<span class="sourceLineNo">066</span>import org.junit.AfterClass;<a name="line.66"></a>
-<span class="sourceLineNo">067</span>import org.junit.Before;<a name="line.67"></a>
-<span class="sourceLineNo">068</span>import org.junit.BeforeClass;<a name="line.68"></a>
-<span class="sourceLineNo">069</span>import org.junit.ClassRule;<a name="line.69"></a>
-<span class="sourceLineNo">070</span>import org.junit.Rule;<a name="line.70"></a>
-<span class="sourceLineNo">071</span>import org.junit.Test;<a name="line.71"></a>
-<span class="sourceLineNo">072</span>import org.junit.experimental.categories.Category;<a name="line.72"></a>
-<span class="sourceLineNo">073</span>import org.junit.rules.TestName;<a name="line.73"></a>
-<span class="sourceLineNo">074</span>import org.slf4j.Logger;<a name="line.74"></a>
-<span class="sourceLineNo">075</span>import org.slf4j.LoggerFactory;<a name="line.75"></a>
-<span class="sourceLineNo">076</span><a name="line.76"></a>
-<span class="sourceLineNo">077</span>/**<a name="line.77"></a>
-<span class="sourceLineNo">078</span> * WAL tests that can be reused across providers.<a name="line.78"></a>
-<span class="sourceLineNo">079</span> */<a name="line.79"></a>
-<span class="sourceLineNo">080</span>@Category({RegionServerTests.class, MediumTests.class})<a name="line.80"></a>
-<span class="sourceLineNo">081</span>public class TestWALFactory {<a name="line.81"></a>
-<span class="sourceLineNo">082</span><a name="line.82"></a>
-<span class="sourceLineNo">083</span> @ClassRule<a name="line.83"></a>
-<span class="sourceLineNo">084</span> public static final HBaseClassTestRule CLASS_RULE =<a name="line.84"></a>
-<span class="sourceLineNo">085</span> HBaseClassTestRule.forClass(TestWALFactory.class);<a name="line.85"></a>
-<span class="sourceLineNo">086</span><a name="line.86"></a>
-<span class="sourceLineNo">087</span> private static final Logger LOG = LoggerFactory.getLogger(TestWALFactory.class);<a name="line.87"></a>
-<span class="sourceLineNo">088</span><a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected static Configuration conf;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> private static MiniDFSCluster cluster;<a name="line.90"></a>
-<span class="sourceLineNo">091</span> protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();<a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected static Path hbaseDir;<a name="line.92"></a>
-<span class="sourceLineNo">093</span> protected static Path hbaseWALDir;<a name="line.93"></a>
-<span class="sourceLineNo">094</span><a name="line.94"></a>
-<span class="sourceLineNo">095</span> protected FileSystem fs;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> protected Path dir;<a name="line.96"></a>
-<span class="sourceLineNo">097</span> protected WALFactory wals;<a name="line.97"></a>
-<span class="sourceLineNo">098</span> private ServerName currentServername;<a name="line.98"></a>
-<span class="sourceLineNo">099</span><a name="line.99"></a>
-<span class="sourceLineNo">100</span> @Rule<a name="line.100"></a>
-<span class="sourceLineNo">101</span> public final TestName currentTest = new TestName();<a name="line.101"></a>
-<span class="sourceLineNo">102</span><a name="line.102"></a>
-<span class="sourceLineNo">103</span> @Before<a name="line.103"></a>
-<span class="sourceLineNo">104</span> public void setUp() throws Exception {<a name="line.104"></a>
-<span class="sourceLineNo">105</span> fs = cluster.getFileSystem();<a name="line.105"></a>
-<span class="sourceLineNo">106</span> dir = new Path(hbaseDir, currentTest.getMethodName());<a name="line.106"></a>
-<span class="sourceLineNo">107</span> this.currentServername = ServerName.valueOf(currentTest.getMethodName(), 16010, 1);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> wals = new WALFactory(conf, this.currentServername.toString());<a name="line.108"></a>
-<span class="sourceLineNo">109</span> }<a name="line.109"></a>
-<span class="sourceLineNo">110</span><a name="line.110"></a>
-<span class="sourceLineNo">111</span> @After<a name="line.111"></a>
-<span class="sourceLineNo">112</span> public void tearDown() throws Exception {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> // testAppendClose closes the FileSystem, which will prevent us from closing cleanly here.<a name="line.113"></a>
-<span class="sourceLineNo">114</span> try {<a name="line.114"></a>
-<span class="sourceLineNo">115</span> wals.close();<a name="line.115"></a>
-<span class="sourceLineNo">116</span> } catch (IOException exception) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> LOG.warn("Encountered exception while closing wal factory. If you have other errors, this" +<a name="line.117"></a>
-<span class="sourceLineNo">118</span> " may be the cause. Message: " + exception);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> LOG.debug("Exception details for failure to close wal factory.", exception);<a name="line.119"></a>
-<span class="sourceLineNo">120</span> }<a name="line.120"></a>
-<span class="sourceLineNo">121</span> FileStatus[] entries = fs.listStatus(new Path("/"));<a name="line.121"></a>
-<span class="sourceLineNo">122</span> for (FileStatus dir : entries) {<a name="line.122"></a>
-<span class="sourceLineNo">123</span> fs.delete(dir.getPath(), true);<a name="line.123"></a>
-<span class="sourceLineNo">124</span> }<a name="line.124"></a>
-<span class="sourceLineNo">125</span> }<a name="line.125"></a>
-<span class="sourceLineNo">126</span><a name="line.126"></a>
-<span class="sourceLineNo">127</span> @BeforeClass<a name="line.127"></a>
-<span class="sourceLineNo">128</span> public static void setUpBeforeClass() throws Exception {<a name="line.128"></a>
-<span class="sourceLineNo">129</span> // Make block sizes small.<a name="line.129"></a>
-<span class="sourceLineNo">130</span> TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);<a name="line.130"></a>
-<span class="sourceLineNo">131</span> // needed for testAppendClose()<a name="line.131"></a>
-<span class="sourceLineNo">132</span> // quicker heartbeat interval for faster DN death notification<a name="line.132"></a>
-<span class="sourceLineNo">133</span> TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);<a name="line.133"></a>
-<span class="sourceLineNo">134</span> TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);<a name="line.134"></a>
-<span class="sourceLineNo">135</span> TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> // faster failover with cluster.shutdown();fs.close() idiom<a name="line.137"></a>
-<span class="sourceLineNo">138</span> TEST_UTIL.getConfiguration()<a name="line.138"></a>
-<span class="sourceLineNo">139</span> .setInt("hbase.ipc.client.connect.max.retries", 1);<a name="line.139"></a>
-<span class="sourceLineNo">140</span> TEST_UTIL.getConfiguration().setInt(<a name="line.140"></a>
-<span class="sourceLineNo">141</span> "dfs.client.block.recovery.retries", 1);<a name="line.141"></a>
+<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.util.CommonFSUtils;<a name="line.60"></a>
+<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.util.FSUtils;<a name="line.61"></a>
+<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.62"></a>
+<span class="sourceLineNo">063</span>import org.apache.hadoop.hdfs.DistributedFileSystem;<a name="line.63"></a>
+<span class="sourceLineNo">064</span>import org.apache.hadoop.hdfs.MiniDFSCluster;<a name="line.64"></a>
+<span class="sourceLineNo">065</span>import org.apache.hadoop.hdfs.protocol.HdfsConstants;<a name="line.65"></a>
+<span class="sourceLineNo">066</span>import org.junit.After;<a name="line.66"></a>
+<span class="sourceLineNo">067</span>import org.junit.AfterClass;<a name="line.67"></a>
+<span class="sourceLineNo">068</span>import org.junit.Before;<a name="line.68"></a>
+<span class="sourceLineNo">069</span>import org.junit.BeforeClass;<a name="line.69"></a>
+<span class="sourceLineNo">070</span>import org.junit.ClassRule;<a name="line.70"></a>
+<span class="sourceLineNo">071</span>import org.junit.Rule;<a name="line.71"></a>
+<span class="sourceLineNo">072</span>import org.junit.Test;<a name="line.72"></a>
+<span class="sourceLineNo">073</span>import org.junit.experimental.categories.Category;<a name="line.73"></a>
+<span class="sourceLineNo">074</span>import org.junit.rules.TestName;<a name="line.74"></a>
+<span class="sourceLineNo">075</span>import org.slf4j.Logger;<a name="line.75"></a>
+<span class="sourceLineNo">076</span>import org.slf4j.LoggerFactory;<a name="line.76"></a>
+<span class="sourceLineNo">077</span><a name="line.77"></a>
+<span class="sourceLineNo">078</span>/**<a name="line.78"></a>
+<span class="sourceLineNo">079</span> * WAL tests that can be reused across providers.<a name="line.79"></a>
+<span class="sourceLineNo">080</span> */<a name="line.80"></a>
+<span class="sourceLineNo">081</span>@Category({RegionServerTests.class, MediumTests.class})<a name="line.81"></a>
+<span class="sourceLineNo">082</span>public class TestWALFactory {<a name="line.82"></a>
+<span class="sourceLineNo">083</span><a name="line.83"></a>
+<span class="sourceLineNo">084</span> @ClassRule<a name="line.84"></a>
+<span class="sourceLineNo">085</span> public static final HBaseClassTestRule CLASS_RULE =<a name="line.85"></a>
+<span class="sourceLineNo">086</span> HBaseClassTestRule.forClass(TestWALFactory.class);<a name="line.86"></a>
+<span class="sourceLineNo">087</span><a name="line.87"></a>
+<span class="sourceLineNo">088</span> private static final Logger LOG = LoggerFactory.getLogger(TestWALFactory.class);<a name="line.88"></a>
+<span class="sourceLineNo">089</span><a name="line.89"></a>
+<span class="sourceLineNo">090</span> protected static Configuration conf;<a name="line.90"></a>
+<span class="sourceLineNo">091</span> private static MiniDFSCluster cluster;<a name="line.91"></a>
+<span class="sourceLineNo">092</span> protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();<a name="line.92"></a>
+<span class="sourceLineNo">093</span> protected static Path hbaseDir;<a name="line.93"></a>
+<span class="sourceLineNo">094</span> protected static Path hbaseWALDir;<a name="line.94"></a>
+<span class="sourceLineNo">095</span><a name="line.95"></a>
+<span class="sourceLineNo">096</span> protected FileSystem fs;<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected Path dir;<a name="line.97"></a>
+<span class="sourceLineNo">098</span> protected WALFactory wals;<a name="line.98"></a>
+<span class="sourceLineNo">099</span> private ServerName currentServername;<a name="line.99"></a>
+<span class="sourceLineNo">100</span><a name="line.100"></a>
+<span class="sourceLineNo">101</span> @Rule<a name="line.101"></a>
+<span class="sourceLineNo">102</span> public final TestName currentTest = new TestName();<a name="line.102"></a>
+<span class="sourceLineNo">103</span><a name="line.103"></a>
+<span class="sourceLineNo">104</span> @Before<a name="line.104"></a>
+<span class="sourceLineNo">105</span> public void setUp() throws Exception {<a name="line.105"></a>
+<span class="sourceLineNo">106</span> fs = cluster.getFileSystem();<a name="line.106"></a>
+<span class="sourceLineNo">107</span> dir = new Path(hbaseDir, currentTest.getMethodName());<a name="line.107"></a>
+<span class="sourceLineNo">108</span> this.currentServername = ServerName.valueOf(currentTest.getMethodName(), 16010, 1);<a name="line.108"></a>
+<span class="sourceLineNo">109</span> wals = new WALFactory(conf, this.currentServername.toString());<a name="line.109"></a>
+<span class="sourceLineNo">110</span> }<a name="line.110"></a>
+<span class="sourceLineNo">111</span><a name="line.111"></a>
+<span class="sourceLineNo">112</span> @After<a name="line.112"></a>
+<span class="sourceLineNo">113</span> public void tearDown() throws Exception {<a name="line.113"></a>
+<span class="sourceLineNo">114</span> // testAppendClose closes the FileSystem, which will prevent us from closing cleanly here.<a name="line.114"></a>
+<span class="sourceLineNo">115</span> try {<a name="line.115"></a>
+<span class="sourceLineNo">116</span> wals.close();<a name="line.116"></a>
+<span class="sourceLineNo">117</span> } catch (IOException exception) {<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Encountered exception while closing wal factory. If you have other errors, this" +<a name="line.118"></a>
+<span class="sourceLineNo">119</span> " may be the cause. Message: " + exception);<a name="line.119"></a>
+<span class="sourceLineNo">120</span> LOG.debug("Exception details for failure to close wal factory.", exception);<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span> FileStatus[] entries = fs.listStatus(new Path("/"));<a name="line.122"></a>
+<span class="sourceLineNo">123</span> for (FileStatus dir : entries) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> fs.delete(dir.getPath(), true);<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span> }<a name="line.126"></a>
+<span class="sourceLineNo">127</span><a name="line.127"></a>
+<span class="sourceLineNo">128</span> @BeforeClass<a name="line.128"></a>
+<span class="sourceLineNo">129</span> public static void setUpBeforeClass() throws Exception {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> CommonFSUtils.setWALRootDir(TEST_UTIL.getConfiguration(), new Path("file:///tmp/wal"));<a name="line.130"></a>
+<span class="sourceLineNo">131</span> // Make block sizes small.<a name="line.131"></a>
+<span class="sourceLineNo">132</span> TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);<a name="line.132"></a>
+<span class="sourceLineNo">133</span> // needed for testAppendClose()<a name="line.133"></a>
+<span class="sourceLineNo">134</span> // quicker heartbeat interval for faster DN death notification<a name="line.134"></a>
+<span class="sourceLineNo">135</span> TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);<a name="line.135"></a>
+<span class="sourceLineNo">136</span> TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);<a name="line.136"></a>
+<span class="sourceLineNo">137</span> TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);<a name="line.137"></a>
+<span class="sourceLineNo">138</span><a name="line.138"></a>
+<span class="sourceLineNo">139</span> // faster failover with cluster.shutdown();fs.close() idiom<a name="line.139"></a>
+<span class="sourceLineNo">140</span> TEST_UTIL.getConfiguration()<a name="line.140"></a>
+<span class="sourceLineNo">141</span> .setInt("hbase.ipc.client.connect.max.retries", 1);<a name="line.141"></a>
<span class="sourceLineNo">142</span> TEST_UTIL.getConfiguration().setInt(<a name="line.142"></a>
-<span class="sourceLineNo">143</span> "hbase.ipc.client.connection.maxidletime", 500);<a name="line.143"></a>
-<span class="sourceLineNo">144</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.timeout", 10000);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 1000);<a name="line.145"></a>
-<span class="sourceLineNo">146</span> TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,<a name="line.146"></a>
-<span class="sourceLineNo">147</span> SampleRegionWALCoprocessor.class.getName());<a name="line.147"></a>
-<span class="sourceLineNo">148</span> TEST_UTIL.startMiniDFSCluster(3);<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span> conf = TEST_UTIL.getConfiguration();<a name="line.150"></a>
-<span class="sourceLineNo">151</span> cluster = TEST_UTIL.getDFSCluster();<a name="line.151"></a>
-<span class="sourceLineNo">152</span><a name="line.152"></a>
-<span class="sourceLineNo">153</span> hbaseDir = TEST_UTIL.createRootDir();<a name="line.153"></a>
-<span class="sourceLineNo">154</span> hbaseWALDir = TEST_UTIL.createWALRootDir();<a name="line.154"></a>
-<span class="sourceLineNo">155</span> }<a name="line.155"></a>
-<span class="sourceLineNo">156</span><a name="line.156"></a>
-<span class="sourceLineNo">157</span> @AfterClass<a name="line.157"></a>
-<span class="sourceLineNo">158</span> public static void tearDownAfterClass() throws Exception {<a name="line.158"></a>
-<span class="sourceLineNo">159</span> TEST_UTIL.shutdownMiniCluster();<a name="line.159"></a>
-<span class="sourceLineNo">160</span> }<a name="line.160"></a>
-<span class="sourceLineNo">161</span><a name="line.161"></a>
-<span class="sourceLineNo">162</span> @Test<a name="line.162"></a>
-<span class="sourceLineNo">163</span> public void canCloseSingleton() throws IOException {<a name="line.163"></a>
-<span class="sourceLineNo">164</span> WALFactory.getInstance(conf).close();<a name="line.164"></a>
-<span class="sourceLineNo">165</span> }<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> /**<a name="line.167"></a>
-<span class="sourceLineNo">168</span> * Just write multiple logs then split. Before fix for HADOOP-2283, this<a name="line.168"></a>
-<span class="sourceLineNo">169</span> * would fail.<a name="line.169"></a>
-<span class="sourceLineNo">170</span> * @throws IOException<a name="line.170"></a>
-<span class="sourceLineNo">171</span> */<a name="line.171"></a>
-<span class="sourceLineNo">172</span> @Test<a name="line.172"></a>
-<span class="sourceLineNo">173</span> public void testSplit() throws IOException {<a name="line.173"></a>
-<span class="sourceLineNo">174</span> final TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.174"></a>
-<span class="sourceLineNo">175</span> final byte [] rowName = tableName.getName();<a name="line.175"></a>
-<span class="sourceLineNo">176</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.176"></a>
-<span class="sourceLineNo">177</span> final int howmany = 3;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> RegionInfo[] infos = new RegionInfo[3];<a name="line.178"></a>
-<span class="sourceLineNo">179</span> Path tabledir = FSUtils.getTableDir(hbaseWALDir, tableName);<a name="line.179"></a>
-<span class="sourceLineNo">180</span> fs.mkdirs(tabledir);<a name="line.180"></a>
-<span class="sourceLineNo">181</span> for (int i = 0; i < howmany; i++) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> infos[i] = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("" + i))<a name="line.182"></a>
-<span class="sourceLineNo">183</span> .setEndKey(Bytes.toBytes("" + (i + 1))).build();<a name="line.183"></a>
-<span class="sourceLineNo">184</span> fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));<a name="line.184"></a>
-<span class="sourceLineNo">185</span> LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());<a name="line.185"></a>
-<span class="sourceLineNo">186</span> }<a name="line.186"></a>
-<span class="sourceLineNo">187</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.187"></a>
-<span class="sourceLineNo">188</span> scopes.put(Bytes.toBytes("column"), 0);<a name="line.188"></a>
-<span class="sourceLineNo">189</span><a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span> // Add edits for three regions.<a name="line.191"></a>
-<span class="sourceLineNo">192</span> for (int ii = 0; ii < howmany; ii++) {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> for (int i = 0; i < howmany; i++) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> final WAL log =<a name="line.194"></a>
-<span class="sourceLineNo">195</span> wals.getWAL(infos[i]);<a name="line.195"></a>
-<span class="sourceLineNo">196</span> for (int j = 0; j < howmany; j++) {<a name="line.196"></a>
-<span class="sourceLineNo">197</span> WALEdit edit = new WALEdit();<a name="line.197"></a>
-<span class="sourceLineNo">198</span> byte [] family = Bytes.toBytes("column");<a name="line.198"></a>
-<span class="sourceLineNo">199</span> byte [] qualifier = Bytes.toBytes(Integer.toString(j));<a name="line.199"></a>
-<span class="sourceLineNo">200</span> byte [] column = Bytes.toBytes("column:" + Integer.toString(j));<a name="line.200"></a>
-<span class="sourceLineNo">201</span> edit.add(new KeyValue(rowName, family, qualifier,<a name="line.201"></a>
-<span class="sourceLineNo">202</span> System.currentTimeMillis(), column));<a name="line.202"></a>
-<span class="sourceLineNo">203</span> LOG.info("Region " + i + ": " + edit);<a name="line.203"></a>
-<span class="sourceLineNo">204</span> WALKeyImpl walKey = new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName,<a name="line.204"></a>
-<span class="sourceLineNo">205</span> System.currentTimeMillis(), mvcc, scopes);<a name="line.205"></a>
-<span class="sourceLineNo">206</span> log.append(infos[i], walKey, edit, true);<a name="line.206"></a>
-<span class="sourceLineNo">207</span> walKey.getWriteEntry();<a name="line.207"></a>
-<span class="sourceLineNo">208</span> }<a name="line.208"></a>
-<span class="sourceLineNo">209</span> log.sync();<a name="line.209"></a>
-<span class="sourceLineNo">210</span> log.rollWriter(true);<a name="line.210"></a>
-<span class="sourceLineNo">211</span> }<a name="line.211"></a>
-<span class="sourceLineNo">212</span> }<a name="line.212"></a>
-<span class="sourceLineNo">213</span> wals.shutdown();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> // The below calculation of logDir relies on insider information... WALSplitter should be connected better<a name="line.214"></a>
-<span class="sourceLineNo">215</span> // with the WAL system.... not requiring explicit path. The oldLogDir is just made up not used.<a name="line.215"></a>
-<span class="sourceLineNo">216</span> Path logDir =<a name="line.216"></a>
-<span class="sourceLineNo">217</span> new Path(new Path(hbaseWALDir, HConstants.HREGION_LOGDIR_NAME),<a name="line.217"></a>
-<span class="sourceLineNo">218</span> this.currentServername.toString());<a name="line.218"></a>
-<span class="sourceLineNo">219</span> Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.219"></a>
-<span class="sourceLineNo">220</span> List<Path> splits = WALSplitter.split(hbaseWALDir, logDir, oldLogDir, fs, conf, wals);<a name="line.220"></a>
-<span class="sourceLineNo">221</span> verifySplits(splits, howmany);<a name="line.221"></a>
-<span class="sourceLineNo">222</span> }<a name="line.222"></a>
-<span class="sourceLineNo">223</span><a name="line.223"></a>
-<span class="sourceLineNo">224</span> /**<a name="line.224"></a>
-<span class="sourceLineNo">225</span> * Test new HDFS-265 sync.<a name="line.225"></a>
-<span class="sourceLineNo">226</span> * @throws Exception<a name="line.226"></a>
-<span class="sourceLineNo">227</span> */<a name="line.227"></a>
-<span class="sourceLineNo">228</span> @Test<a name="line.228"></a>
-<span class="sourceLineNo">229</span> public void Broken_testSync() throws Exception {<a name="line.229"></a>
-<span class="sourceLineNo">230</span> TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.230"></a>
-<span class="sourceLineNo">231</span> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.231"></a>
-<span class="sourceLineNo">232</span> // First verify that using streams all works.<a name="line.232"></a>
-<span class="sourceLineNo">233</span> Path p = new Path(dir, currentTest.getMethodName() + ".fsdos");<a name="line.233"></a>
-<span class="sourceLineNo">234</span> FSDataOutputStream out = fs.create(p);<a name="line.234"></a>
-<span class="sourceLineNo">235</span> out.write(tableName.getName());<a name="line.235"></a>
-<span class="sourceLineNo">236</span> Method syncMethod = null;<a name="line.236"></a>
-<span class="sourceLineNo">237</span> try {<a name="line.237"></a>
-<span class="sourceLineNo">238</span> syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});<a name="line.238"></a>
-<span class="sourceLineNo">239</span> } catch (NoSuchMethodException e) {<a name="line.239"></a>
-<span class="sourceLineNo">240</span> try {<a name="line.240"></a>
-<span class="sourceLineNo">241</span> syncMethod = out.getClass().getMethod("sync", new Class<?> []{});<a name="line.241"></a>
-<span class="sourceLineNo">242</span> } catch (NoSuchMethodException ex) {<a name="line.242"></a>
-<span class="sourceLineNo">243</span> fail("This version of Hadoop supports neither Syncable.sync() " +<a name="line.243"></a>
-<span class="sourceLineNo">244</span> "nor Syncable.hflush().");<a name="line.244"></a>
-<span class="sourceLineNo">245</span> }<a name="line.245"></a>
-<span class="sourceLineNo">246</span> }<a name="line.246"></a>
-<span class="sourceLineNo">247</span> syncMethod.invoke(out, new Object[]{});<a name="line.247"></a>
-<span class="sourceLineNo">248</span> FSDataInputStream in = fs.open(p);<a name="line.248"></a>
-<span class="sourceLineNo">249</span> assertTrue(in.available() > 0);<a name="line.249"></a>
-<span class="sourceLineNo">250</span> byte [] buffer = new byte [1024];<a name="line.250"></a>
-<span class="sourceLineNo">251</span> int read = in.read(buffer);<a name="line.251"></a>
-<span class="sourceLineNo">252</span> assertEquals(tableName.getName().length, read);<a name="line.252"></a>
-<span class="sourceLineNo">253</span> out.close();<a name="line.253"></a>
-<span class="sourceLineNo">254</span> in.close();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> final int total = 20;<a name="line.256"></a>
-<span class="sourceLineNo">257</span> WAL.Reader reader = null;<a name="line.257"></a>
-<span class="sourceLineNo">258</span><a name="line.258"></a>
-<span class="sourceLineNo">259</span> try {<a name="line.259"></a>
-<span class="sourceLineNo">260</span> RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.260"></a>
-<span class="sourceLineNo">261</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.261"></a>
-<span class="sourceLineNo">262</span> scopes.put(tableName.getName(), 0);<a name="line.262"></a>
-<span class="sourceLineNo">263</span> final WAL wal = wals.getWAL(info);<a name="line.263"></a>
-<span class="sourceLineNo">264</span><a name="line.264"></a>
-<span class="sourceLineNo">265</span> for (int i = 0; i < total; i++) {<a name="line.265"></a>
-<span class="sourceLineNo">266</span> WALEdit kvs = new WALEdit();<a name="line.266"></a>
-<span class="sourceLineNo">267</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.267"></a>
-<span class="sourceLineNo">268</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.268"></a>
-<span class="sourceLineNo">269</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.269"></a>
-<span class="sourceLineNo">270</span> }<a name="line.270"></a>
-<span class="sourceLineNo">271</span> // Now call sync and try reading. Opening a Reader before you sync just<a name="line.271"></a>
-<span class="sourceLineNo">272</span> // gives you EOFE.<a name="line.272"></a>
-<span class="sourceLineNo">273</span> wal.sync();<a name="line.273"></a>
-<span class="sourceLineNo">274</span> // Open a Reader.<a name="line.274"></a>
-<span class="sourceLineNo">275</span> Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);<a name="line.275"></a>
-<span class="sourceLineNo">276</span> reader = wals.createReader(fs, walPath);<a name="line.276"></a>
-<span class="sourceLineNo">277</span> int count = 0;<a name="line.277"></a>
-<span class="sourceLineNo">278</span> WAL.Entry entry = new WAL.Entry();<a name="line.278"></a>
-<span class="sourceLineNo">279</span> while ((entry = reader.next(entry)) != null) count++;<a name="line.279"></a>
-<span class="sourceLineNo">280</span> assertEquals(total, count);<a name="line.280"></a>
-<span class="sourceLineNo">281</span> reader.close();<a name="line.281"></a>
-<span class="sourceLineNo">282</span> // Add test that checks to see that an open of a Reader works on a file<a name="line.282"></a>
-<span class="sourceLineNo">283</span> // that has had a sync done on it.<a name="line.283"></a>
-<span class="sourceLineNo">284</span> for (int i = 0; i < total; i++) {<a name="line.284"></a>
-<span class="sourceLineNo">285</span> WALEdit kvs = new WALEdit();<a name="line.285"></a>
-<span class="sourceLineNo">286</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.286"></a>
-<span class="sourceLineNo">287</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.287"></a>
-<span class="sourceLineNo">288</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.288"></a>
-<span class="sourceLineNo">289</span> }<a name="line.289"></a>
-<span class="sourceLineNo">290</span> wal.sync();<a name="line.290"></a>
-<span class="sourceLineNo">291</span> reader = wals.createReader(fs, walPath);<a name="line.291"></a>
-<span class="sourceLineNo">292</span> count = 0;<a name="line.292"></a>
-<span class="sourceLineNo">293</span> while((entry = reader.next(entry)) != null) count++;<a name="line.293"></a>
-<span class="sourceLineNo">294</span> assertTrue(count >= total);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> reader.close();<a name="line.295"></a>
-<span class="sourceLineNo">296</span> // If I sync, should see double the edits.<a name="line.296"></a>
-<span class="sourceLineNo">297</span> wal.sync();<a name="line.297"></a>
-<span class="sourceLineNo">298</span> reader = wals.createReader(fs, walPath);<a name="line.298"></a>
-<span class="sourceLineNo">299</span> count = 0;<a name="line.299"></a>
-<span class="sourceLineNo">300</span> while((entry = reader.next(entry)) != null) count++;<a name="line.300"></a>
-<span class="sourceLineNo">301</span> assertEquals(total * 2, count);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> reader.close();<a name="line.302"></a>
-<span class="sourceLineNo">303</span> // Now do a test that ensures stuff works when we go over block boundary,<a name="line.303"></a>
-<span class="sourceLineNo">304</span> // especially that we return good length on file.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> final byte [] value = new byte[1025 * 1024]; // Make a 1M value.<a name="line.305"></a>
-<span class="sourceLineNo">306</span> for (int i = 0; i < total; i++) {<a name="line.306"></a>
-<span class="sourceLineNo">307</span> WALEdit kvs = new WALEdit();<a name="line.307"></a>
-<span class="sourceLineNo">308</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value));<a name="line.308"></a>
-<span class="sourceLineNo">309</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.309"></a>
-<span class="sourceLineNo">310</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> }<a name="line.311"></a>
-<span class="sourceLineNo">312</span> // Now I should have written out lots of blocks. Sync then read.<a name="line.312"></a>
-<span class="sourceLineNo">313</span> wal.sync();<a name="line.313"></a>
-<span class="sourceLineNo">314</span> reader = wals.createReader(fs, walPath);<a name="line.314"></a>
-<span class="sourceLineNo">315</span> count = 0;<a name="line.315"></a>
-<span class="sourceLineNo">316</span> while((entry = reader.next(entry)) != null) count++;<a name="line.316"></a>
-<span class="sourceLineNo">317</span> assertEquals(total * 3, count);<a name="line.317"></a>
-<span class="sourceLineNo">318</span> reader.close();<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // shutdown and ensure that Reader gets right length also.<a name="line.319"></a>
-<span class="sourceLineNo">320</span> wal.shutdown();<a name="line.320"></a>
-<span class="sourceLineNo">321</span> reader = wals.createReader(fs, walPath);<a name="line.321"></a>
-<span class="sourceLineNo">322</span> count = 0;<a name="line.322"></a>
-<span class="sourceLineNo">323</span> while((entry = reader.next(entry)) != null) count++;<a name="line.323"></a>
-<span class="sourceLineNo">324</span> assertEquals(total * 3, count);<a name="line.324"></a>
-<span class="sourceLineNo">325</span> reader.close();<a name="line.325"></a>
-<span class="sourceLineNo">326</span> } finally {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> if (reader != null) reader.close();<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span> }<a name="line.329"></a>
-<span class="sourceLineNo">330</span><a name="line.330"></a>
-<span class="sourceLineNo">331</span> private void verifySplits(final List<Path> splits, final int howmany)<a name="line.331"></a>
-<span class="sourceLineNo">332</span> throws IOException {<a name="line.332"></a>
-<span class="sourceLineNo">333</span> assertEquals(howmany * howmany, splits.size());<a name="line.333"></a>
-<span class="sourceLineNo">334</span> for (int i = 0; i < splits.size(); i++) {<a name="line.334"></a>
-<span class="sourceLineNo">335</span> LOG.info("Verifying=" + splits.get(i));<a name="line.335"></a>
-<span class="sourceLineNo">336</span> WAL.Reader reader = wals.createReader(fs, splits.get(i));<a name="line.336"></a>
-<span class="sourceLineNo">337</span> try {<a name="line.337"></a>
-<span class="sourceLineNo">338</span> int count = 0;<a name="line.338"></a>
-<span class="sourceLineNo">339</span> String previousRegion = null;<a name="line.339"></a>
-<span class="sourceLineNo">340</span> long seqno = -1;<a name="line.340"></a>
-<span class="sourceLineNo">341</span> WAL.Entry entry = new WAL.Entry();<a name="line.341"></a>
-<span class="sourceLineNo">342</span> while((entry = reader.next(entry)) != null) {<a name="line.342"></a>
-<span class="sourceLineNo">343</span> WALKey key = entry.getKey();<a name="line.343"></a>
-<span class="sourceLineNo">344</span> String region = Bytes.toString(key.getEncodedRegionName());<a name="line.344"></a>
-<span class="sourceLineNo">345</span> // Assert that all edits are for same region.<a name="line.345"></a>
-<span class="sourceLineNo">346</span> if (previousRegion != null) {<a name="line.346"></a>
-<span class="sourceLineNo">347</span> assertEquals(previousRegion, region);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span> LOG.info("oldseqno=" + seqno + ", newseqno=" + key.getSequenceId());<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assertTrue(seqno < key.getSequenceId());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> seqno = key.getSequenceId();<a name="line.351"></a>
-<span class="sourceLineNo">352</span> previousRegion = region;<a name="line.352"></a>
-<span class="sourceLineNo">353</span> count++;<a name="line.353"></a>
-<span class="sourceLineNo">354</span> }<a name="line.354"></a>
-<span class="sourceLineNo">355</span> assertEquals(howmany, count);<a name="line.355"></a>
-<span class="sourceLineNo">356</span> } finally {<a name="line.356"></a>
-<span class="sourceLineNo">357</span> reader.close();<a name="line.357"></a>
-<span class="sourceLineNo">358</span> }<a name="line.358"></a>
-<span class="sourceLineNo">359</span> }<a name="line.359"></a>
-<span class="sourceLineNo">360</span> }<a name="line.360"></a>
-<span class="sourceLineNo">361</span><a name="line.361"></a>
-<span class="sourceLineNo">362</span> /*<a name="line.362"></a>
-<span class="sourceLineNo">363</span> * We pass different values to recoverFileLease() so that different code paths are covered<a name="line.363"></a>
-<span class="sourceLineNo">364</span> *<a name="line.364"></a>
-<span class="sourceLineNo">365</span> * For this test to pass, requires:<a name="line.365"></a>
-<span class="sourceLineNo">366</span> * 1. HDFS-200 (append support)<a name="line.366"></a>
-<span class="sourceLineNo">367</span> * 2. HDFS-988 (SafeMode should freeze file operations<a name="line.367"></a>
-<span class="sourceLineNo">368</span> * [FSNamesystem.nextGenerationStampForBlock])<a name="line.368"></a>
-<span class="sourceLineNo">369</span> * 3. HDFS-142 (on restart, maintain pendingCreates)<a name="line.369"></a>
-<span class="sourceLineNo">370</span> */<a name="line.370"></a>
-<span class="sourceLineNo">371</span> @Test<a name="line.371"></a>
-<span class="sourceLineNo">372</span> public void testAppendClose() throws Exception {<a name="line.372"></a>
-<span class="sourceLineNo">373</span> TableName tableName =<a name="line.373"></a>
-<span class="sourceLineNo">374</span> TableName.valueOf(currentTest.getMethodName());<a name="line.374"></a>
-<span class="sourceLineNo">375</span> RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.375"></a>
-<span class="sourceLineNo">376</span><a name="line.376"></a>
-<span class="sourceLineNo">377</span> WAL wal = wals.getWAL(regionInfo);<a name="line.377"></a>
-<span class="sourceLineNo">378</span> int total = 20;<a name="line.378"></a>
-<span class="sourceLineNo">379</span><a name="line.379"></a>
-<span class="sourceLineNo">380</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.380"></a>
-<span class="sourceLineNo">381</span> scopes.put(tableName.getName(), 0);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();<a name="line.382"></a>
-<span class="sourceLineNo">383</span> for (int i = 0; i < total; i++) {<a name="line.383"></a>
-<span class="sourceLineNo">384</span> WALEdit kvs = new WALEdit();<a name="line.384"></a>
-<span class="sourceLineNo">385</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.385"></a>
-<span class="sourceLineNo">386</span> wal.append(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,<a name="line.386"></a>
-<span class="sourceLineNo">387</span> System.currentTimeMillis(), mvcc, scopes),<a name="line.387"></a>
-<span class="sourceLineNo">388</span> kvs, true);<a name="line.388"></a>
-<span class="sourceLineNo">389</span> }<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // Now call sync to send the data to HDFS datanodes<a name="line.390"></a>
-<span class="sourceLineNo">391</span> wal.sync();<a name="line.391"></a>
-<span class="sourceLineNo">392</span> int namenodePort = cluster.getNameNodePort();<a name="line.392"></a>
-<span class="sourceLineNo">393</span> final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);<a name="line.393"></a>
-<span class="sourceLineNo">394</span><a name="line.394"></a>
-<span class="sourceLineNo">395</span><a name="line.395"></a>
-<span class="sourceLineNo">396</span> // Stop the cluster. (ensure restart since we're sharing MiniDFSCluster)<a name="line.396"></a>
-<span class="sourceLineNo">397</span> try {<a name="line.397"></a>
-<span class="sourceLineNo">398</span> DistributedFileSystem dfs = cluster.getFileSystem();<a name="line.398"></a>
-<span class="sourceLineNo">399</span> dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> TEST_UTIL.shutdownMiniDFSCluster();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> try {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> // wal.writer.close() will throw an exception,<a name="line.402"></a>
-<span class="sourceLineNo">403</span> // but still call this since it closes the LogSyncer thread first<a name="line.403"></a>
-<span class="sourceLineNo">404</span> wal.shutdown();<a name="line.404"></a>
-<span class="sourceLineNo">405</span> } catch (IOException e) {<a name="line.405"></a>
-<span class="sourceLineNo">406</span> LOG.info(e.toString(), e);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> }<a name="line.407"></a>
-<span class="sourceLineNo">408</span> fs.close(); // closing FS last so DFSOutputStream can't call close<a name="line.408"></a>
-<span class="sourceLineNo">409</span> LOG.info("STOPPED first instance of the cluster");<a name="line.409"></a>
-<span class="sourceLineNo">410</span> } finally {<a name="line.410"></a>
-<span class="sourceLineNo">411</span> // Restart the cluster<a name="line.411"></a>
-<span class="sourceLineNo">412</span> while (cluster.isClusterUp()){<a name="line.412"></a>
-<span class="sourceLineNo">413</span> LOG.error("Waiting for cluster to go down");<a name="line.413"></a>
-<span class="sourceLineNo">414</span> Thread.sleep(1000);<a name="line.414"></a>
-<span class="sourceLineNo">415</span> }<a name="line.415"></a>
-<span class="sourceLineNo">416</span> assertFalse(cluster.isClusterUp());<a name="line.416"></a>
-<span class="sourceLineNo">417</span> cluster = null;<a name="line.417"></a>
-<span class="sourceLineNo">418</span> for (int i = 0; i < 100; i++) {<a name="line.418"></a>
-<span class="sourceLineNo">419</span> try {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> cluster = TEST_UTIL.startMiniDFSClusterForTestWAL(namenodePort);<a name="line.420"></a>
-<span class="sourceLineNo">421</span> break;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> } catch (BindException e) {<a name="line.422"></a>
-<span class="sourceLineNo">423</span> LOG.info("Sleeping. BindException bringing up new cluster");<a name="line.423"></a>
-<span class="sourceLineNo">424</span> Threads.sleep(1000);<a name="line.424"></a>
-<span class="sourceLineNo">425</span> }<a name="line.425"></a>
-<span class="sourceLineNo">426</span> }<a name="line.426"></a>
-<span class="sourceLineNo">427</span> cluster.waitActive();<a name="line.427"></a>
-<span class="sourceLineNo">428</span> fs = cluster.getFileSystem();<a name="line.428"></a>
-<span class="sourceLineNo">429</span> LOG.info("STARTED second instance.");<a name="line.429"></a>
-<span class="sourceLineNo">430</span> }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span> // set the lease period to be 1 second so that the<a name="line.432"></a>
-<span class="sourceLineNo">433</span> // namenode triggers lease recovery upon append request<a name="line.433"></a>
-<span class="sourceLineNo">434</span> Method setLeasePeriod = cluster.getClass()<a name="line.434"></a>
-<span class="sourceLineNo">435</span> .getDeclaredMethod("setLeasePeriod", new Class[]{Long.TYPE, Long.TYPE});<a name="line.435"></a>
-<span class="sourceLineNo">436</span> setLeasePeriod.setAccessible(true);<a name="line.436"></a>
-<span class="sourceLineNo">437</span> setLeasePeriod.invoke(cluster, 1000L, 1000L);<a name="line.437"></a>
-<span class="sourceLineNo">438</span> try {<a name="line.438"></a>
-<span class="sourceLineNo">439</span> Thread.sleep(1000);<a name="line.439"></a>
-<span class="sourceLineNo">440</span> } catch (InterruptedException e) {<a name="line.440"></a>
-<span class="sourceLineNo">441</span> LOG.info(e.toString(), e);<a name="line.441"></a>
-<span class="sourceLineNo">442</span> }<a name="line.442"></a>
-<span class="sourceLineNo">443</span><a name="line.443"></a>
-<span class="sourceLineNo">444</span> // Now try recovering the log, like the HMaster would do<a name="line.444"></a>
-<span class="sourceLineNo">445</span> final FileSystem recoveredFs = fs;<a name="line.445"></a>
-<span class="sourceLineNo">446</span> final Configuration rlConf = conf;<a name="line.446"></a>
-<span class="sourceLineNo">447</span><a name="line.447"></a>
-<span class="sourceLineNo">448</span> class RecoverLogThread extends Thread {<a name="line.448"></a>
-<span class="sourceLineNo">449</span> public Exception exception = null;<a name="line.449"></a>
-<span class="sourceLineNo">450</span> @Override<a name="line.450"></a>
-<span class="sourceLineNo">451</span> public void run() {<a name="line.451"></a>
-<span class="sourceLineNo">452</span> try {<a name="line.452"></a>
-<span class="sourceLineNo">453</span> FSUtils.getInstance(fs, rlConf)<a name="line.453"></a>
-<span class="sourceLineNo">454</span> .recoverFileLease(recoveredFs, walPath, rlConf, null);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> } catch (IOException e) {<a name="line.455"></a>
-<span class="sourceLineNo">456</span> exception = e;<a name="line.456"></a>
-<span class="sourceLineNo">457</span> }<a name="line.457"></a>
-<span class="sourceLineNo">458</span> }<a name="line.458"></a>
-<span class="sourceLineNo">459</span> }<a name="line.459"></a>
-<span class="sourceLineNo">460</span><a name="line.460"></a>
-<span class="sourceLineNo">461</span> RecoverLogThread t = new RecoverLogThread();<a name="line.461"></a>
-<span class="sourceLineNo">462</span> t.start();<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // Timeout after 60 sec. Without correct patches, would be an infinite loop<a name="line.463"></a>
-<span class="sourceLineNo">464</span> t.join(60 * 1000);<a name="line.464"></a>
-<span class="sourceLineNo">465</span> if(t.isAlive()) {<a name="line.465"></a>
-<span class="sourceLineNo">466</span> t.interrupt();<a name="line.466"></a>
-<span class="sourceLineNo">467</span> throw new Exception("Timed out waiting for WAL.recoverLog()");<a name="line.467"></a>
-<span class="sourceLineNo">468</span> }<a name="line.468"></a>
-<span class="sourceLineNo">469</span><a name="line.469"></a>
-<span class="sourceLineNo">470</span> if (t.exception != null)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throw t.exception;<a name="line.471"></a>
-<span class="sourceLineNo">472</span><a name="line.472"></a>
-<span class="sourceLineNo">473</span> // Make sure you can read all the content<a name="line.473"></a>
-<span class="sourceLineNo">474</span> WAL.Reader reader = wals.createReader(fs, walPath);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> int count = 0;<a name="line.475"></a>
-<span class="sourceLineNo">476</span> WAL.Entry entry = new WAL.Entry();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> while (reader.next(entry) != null) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> count++;<a name="line.478"></a>
-<span class="sourceLineNo">479</span> assertTrue("Should be one KeyValue per WALEdit",<a name="line.479"></a>
-<span class="sourceLineNo">480</span> entry.getEdit().getCells().size() == 1);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> }<a name="line.481"></a>
-<span class="sourceLineNo">482</span> assertEquals(total, count);<a name="line.482"></a>
-<span class="sourceLineNo">483</span> reader.close();<a name="line.483"></a>
-<span class="sourceLineNo">484</span><a name="line.484"></a>
-<span class="sourceLineNo">485</span> // Reset the lease period<a name="line.485"></a>
-<span class="sourceLineNo">486</span> setLeasePeriod.invoke(cluster, new Object[]{ 60000L, 3600000L });<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span><a name="line.488"></a>
-<span class="sourceLineNo">489</span> /**<a name="line.489"></a>
-<span class="sourceLineNo">490</span> * Tests that we can write out an edit, close, and then read it back in again.<a name="line.490"></a>
-<span class="sourceLineNo">491</span> */<a name="line.491"></a>
-<span class="sourceLineNo">492</span> @Test<a name="line.492"></a>
-<span class="sourceLineNo">493</span> public void testEditAdd() throws IOException {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> int colCount = 10;<a name="line.494"></a>
-<span class="sourceLineNo">495</span> TableDescriptor htd =<a name="line.495"></a>
-<span class="sourceLineNo">496</span> TableDescriptorBuilder.newBuilder(TableName.valueOf(currentTest.getMethodName()))<a name="line.496"></a>
-<span class="sourceLineNo">497</span> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("column")).build();<a name="line.497"></a>
-<span class="sourceLineNo">498</span> NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);<a name="line.498"></a>
-<span class="sourceLineNo">499</span> for (byte[] fam : htd.getColumnFamilyNames()) {<a name="line.499"></a>
-<span class="sourceLineNo">500</span> scopes.put(fam, 0);<a name="line.500"></a>
-<span class="sourceLineNo">501</span> }<a name="line.501"></a>
-<span class="sourceLineNo">502</span> byte[] row = Bytes.toBytes("row");<a name="line.502"></a>
-<span class="sourceLineNo">503</span> WAL.Reader reader = null;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> try {<a name="line.504"></a>
-<span class="sourceLineNo">505</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.505"></a>
-<span class="sourceLineNo">506</span><a name="line.506"></a>
-<span class="sourceLineNo">507</span> // Write columns named 1, 2, 3, etc. and then values of single byte<a name="line.507"></a>
-<span class="sourceLineNo">508</span> // 1, 2, 3...<a name="line.508"></a>
-<span class="sourceLineNo">509</span> long timestamp = System.currentTimeMillis();<a name="line.509"></a>
-<span class="sourceLineNo">510</span> WALEdit cols = new WALEdit();<a name="line.510"></a>
-<span class="sourceLineNo">511</span> for (int i = 0; i < colCount; i++) {<a name="line.511"></a>
-<span class="sourceLineNo">512</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.512"></a>
-<span class="sourceLineNo">513</span> Bytes.toBytes(Integer.toString(i)),<a name="line.513"></a>
-<span class="sourceLineNo">514</span> timestamp, new byte[] { (byte)(i + '0') }));<a name="line.514"></a>
-<span class="sourceLineNo">515</span> }<a name="line.515"></a>
-<span class="sourceLineNo">516</span> RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(row)<a name="line.516"></a>
-<span class="sourceLineNo">517</span> .setEndKey(Bytes.toBytes(Bytes.toString(row) + "1")).build();<a name="line.517"></a>
-<span class="sourceLineNo">518</span> final WAL log = wals.getWAL(info);<a name="line.518"></a>
-<span class="sourceLineNo">519</span><a name="line.519"></a>
-<span class="sourceLineNo">520</span> final long txid = log.append(info,<a name="line.520"></a>
-<span class="sourceLineNo">521</span> new WALKeyImpl(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),<a name="line.521"></a>
-<span class="sourceLineNo">522</span> mvcc, scopes),<a name="line.522"></a>
-<span class="sourceLineNo">523</span> cols, true);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> log.sync(txid);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> log.startCacheFlush(info.getEncodedNameAsBytes(), htd.getColumnFamilyNames());<a name="line.525"></a>
-<span class="sourceLineNo">526</span> log.completeCacheFlush(info.getEncodedNameAsBytes());<a name="line.526"></a>
-<span class="sourceLineNo">527</span> log.shutdown();<a name="line.527"></a>
-<span class="sourceLineNo">528</span> Path filename = AbstractFSWALProvider.getCurrentFileName(log);<a name="line.528"></a>
-<span class="sourceLineNo">529</span> // Now open a reader on the log and assert append worked.<a name="line.529"></a>
-<span class="sourceLineNo">530</span> reader = wals.createReader(fs, filename);<a name="line.530"></a>
-<span class="sourceLineNo">531</span> // Above we added all columns on a single row so we only read one<a name="line.531"></a>
-<span class="sourceLineNo">532</span> // entry in the below... thats why we have '1'.<a name="line.532"></a>
-<span class="sourceLineNo">533</span> for (int i = 0; i < 1; i++) {<a name="line.533"></a>
-<span class="sourceLineNo">534</span> WAL.Entry entry = reader.next(null);<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (entry == null) break;<a name="line.535"></a>
-<span class="sourceLineNo">536</span> WALKey key = entry.getKey();<a name="line.536"></a>
-<span class="sourceLineNo">537</span> WALEdit val = entry.getEdit();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));<a name="line.538"></a>
-<span class="sourceLineNo">539</span> assertTrue(htd.getTableName().equals(key.getTableName()));<a name="line.539"></a>
-<span class="sourceLineNo">540</span> Cell cell = val.getCells().get(0);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> assertTrue(Bytes.equals(row, 0, row.length, cell.getRowArray(), cell.getRowOffset(),<a name="line.541"></a>
-<span class="sourceLineNo">542</span> cell.getRowLength()));<a name="line.542"></a>
-<span class="sourceLineNo">543</span> assertEquals((byte)(i + '0'), CellUtil.cloneValue(cell)[0]);<a name="line.543"></a>
-<span class="sourceLineNo">544</span> System.out.println(key + " " + val);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> } finally {<a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (reader != null) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> reader.close();<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span> }<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> @Test<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public void testAppend() throws IOException {<a name="line.554"></a>
-<span class="sourceLineNo">555</span> int colCount = 10;<a name="line.555"></a>
-<span class="sourceLineNo">556</span> TableDescriptor htd =<a name="line.556"></a>
-<span class="sourceLineNo">557</span> TableDescriptorBuilder.newBuilder(TableName.valueOf(currentTest.getMethodName()))<a name="line.557"></a>
-<span class="sourceLineNo">558</span> .setColumnFamily(ColumnFamilyDescriptorBuilder.of("column")).build();<a name="line.558"></a>
-<span class="sourceLineNo">559</span> NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);<a name="line.559"></a>
-<span class="sourceLineNo">560</span> for (byte[] fam : htd.getColumnFamilyNames()) {<a name="line.560"></a>
-<span class="sourceLineNo">561</span> scopes.put(fam, 0);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> }<a name="line.562"></a>
-<span class="sourceLineNo">563</span> byte[] row = Bytes.toBytes("row");<a name="line.563"></a>
-<span class="sourceLineNo">564</span> WAL.Reader reader = null;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.565"></a>
-<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // Write columns named 1, 2, 3, etc. and then values of single byte<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // 1, 2, 3...<a name="line.568"></a>
-<span class="sourceLineNo">569</span> long timestamp = System.currentTimeMillis();<a name="line.569"></a>
-<span class="sourceLineNo">570</span> WALEdit cols = new WALEdit();<a name="line.570"></a>
-<span class="sourceLineNo">571</span> for (int i = 0; i < colCount; i++) {<a name="line.571"></a>
-<span class="sourceLineNo">572</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.572"></a>
-<span class="sourceLineNo">573</span> Bytes.toBytes(Integer.toString(i)),<a name="line.573"></a>
-<span class="sourceLineNo">574</span> timestamp, new byte[] { (byte)(i + '0') }));<a name="line.574"></a>
-<span class="sourceLineNo">575</span> }<a name="line.575"></a>
-<span class="sourceLineNo">576</span> RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();<a name="line.576"></a>
-<span class="sourceLineNo">577</span> final WAL log = wals.getWAL(hri);<a name="line.577"></a>
-<span class="sourceLineNo">578</span> final long txid = log.append(hri,<a name="line.578"></a>
-<span class="sourceLineNo">579</span> new WALKeyImpl(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),<a name="line.579"></a>
-<span class="sourceLineNo">580</span> mvcc, scopes),<a name="line.580"></a>
-<span class="sourceLineNo">581</span> cols, true);<a name="line.581"></a>
-<span class="sourceLineNo">582</span> log.sync(txid);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getColumnFamilyNames());<a name="line.583"></a>
-<span class="sourceLineNo">584</span> log.completeCacheFlush(hri.getEncodedNameAsBytes());<a name="line.584"></a>
-<span class="sourceLineNo">585</span> log.shutdown();<a name="line.585"></a>
-<span class="sourceLineNo">586</span> Path filename = AbstractFSWALProvider.getCurrentFileName(log);<a name="line.586"></a>
-<span class="sourceLineNo">587</span> // Now open a reader on the log and assert append worked.<a name="line.587"></a>
-<span class="sourceLineNo">588</span> reader = wals.createReader(fs, filename);<a name="line.588"></a>
-<span class="sourceLineNo">589</span> WAL.Entry entry = reader.next();<a name="line.589"></a>
-<span class="sourceLineNo">590</span> assertEquals(colCount, entry.getEdit().size());<a name="line.590"></a>
-<span class="sourceLineNo">591</span> int idx = 0;<a name="line.591"></a>
-<span class="sourceLineNo">592</span> for (Cell val : entry.getEdit().getCells()) {<a name="line.592"></a>
-<span class="sourceLineNo">593</span> assertTrue(Bytes.equals(hri.getEncodedNameAsBytes(),<a name="line.593"></a>
-<span class="sourceLineNo">594</span> entry.getKey().getEncodedRegionName()));<a name="line.594"></a>
-<span class="sourceLineNo">595</span> assertTrue(htd.getTableName().equals(entry.getKey().getTableName()));<a name="line.595"></a>
-<span class="sourceLineNo">596</span> assertTrue(Bytes.equals(row, 0, row.length, val.getRowArray(), val.getRowOffset(),<a name="line.596"></a>
-<span class="sourceLineNo">597</span> val.getRowLength()));<a name="line.597"></a>
-<span class="sourceLineNo">598</span> assertEquals((byte) (idx + '0'), CellUtil.cloneValue(val)[0]);<a name="line.598"></a>
-<span class="sourceLineNo">599</span> System.out.println(entry.getKey() + " " + val);<a name="line.599"></a>
-<span class="sourceLineNo">600</span> idx++;<a name="line.600"></a>
-<span class="sourceLineNo">601</span> }<a name="line.601"></a>
-<span class="sourceLineNo">602</span> } finally {<a name="line.602"></a>
-<span class="sourceLineNo">603</span> if (reader != null) {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> reader.close();<a name="line.604"></a>
-<span class="sourceLineNo">605</span> }<a name="line.605"></a>
-<span class="sourceLineNo">606</span> }<a name="line.606"></a>
-<span class="sourceLineNo">607</span> }<a name="line.607"></a>
-<span class="sourceLineNo">608</span><a name="line.608"></a>
-<span class="sourceLineNo">609</span> /**<a name="line.609"></a>
-<span class="sourceLineNo">610</span> * Test that we can visit entries before they are appended<a name="line.610"></a>
-<span class="sourceLineNo">611</span> * @throws Exception<a name="line.611"></a>
-<span class="sourceLineNo">612</span> */<a name="line.612"></a>
-<span class="sourceLineNo">613</span> @Test<a name="line.613"></a>
-<span class="sourceLineNo">614</span> public void testVisitors() throws Exception {<a name="line.614"></a>
-<span class="sourceLineNo">615</span> final int COL_COUNT = 10;<a name="line.615"></a>
-<span class="sourceLineNo">616</span> final TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.616"></a>
-<span class="sourceLineNo">617</span> final byte [] row = Bytes.toBytes("row");<a name="line.617"></a>
-<span class="sourceLineNo">618</span> final DumbWALActionsListener visitor = new DumbWALActionsListener();<a name="line.618"></a>
-<span class="sourceLineNo">619</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.619"></a>
-<span class="sourceLineNo">620</span> long timestamp = System.currentTimeMillis();<a name="line.620"></a>
-<span class="sourceLineNo">621</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.621"></a>
-<span class="sourceLineNo">622</span> scopes.put(Bytes.toBytes("column"), 0);<a name="line.622"></a>
-<span class="sourceLineNo">623</span><a name="line.623"></a>
-<span class="sourceLineNo">624</span> RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.624"></a>
-<span class="sourceLineNo">625</span> final WAL log = wals.getWAL(hri);<a name="line.625"></a>
-<span class="sourceLineNo">626</span> log.registerWALActionsListener(visitor);<a name="line.626"></a>
-<span class="sourceLineNo">627</span> for (int i = 0; i < COL_COUNT; i++) {<a name="line.627"></a>
-<span class="sourceLineNo">628</span> WALEdit cols = new WALEdit();<a name="line.628"></a>
-<span class="sourceLineNo">629</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.629"></a>
-<span class="sourceLineNo">630</span> Bytes.toBytes(Integer.toString(i)),<a name="line.630"></a>
-<span class="sourceLineNo">631</span> timestamp, new byte[]{(byte) (i + '0')}));<a name="line.631"></a>
-<span class="sourceLineNo">632</span> log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName,<a name="line.632"></a>
-<span class="sourceLineNo">633</span> System.currentTimeMillis(), mvcc, scopes), cols, true);<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> log.sync();<a name="line.635"></a>
-<span class="sourceLineNo">636</span> assertEquals(COL_COUNT, visitor.increments);<a name="line.636"></a>
-<span class="sourceLineNo">637</span> log.unregisterWALActionsListener(visitor);<a name="line.637"></a>
-<span class="sourceLineNo">638</span> WALEdit cols = new WALEdit();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> cols.add(new KeyValue(row, Bytes.toBytes("column"),<a name="line.639"></a>
-<span class="sourceLineNo">640</span> Bytes.toBytes(Integer.toString(11)),<a name="line.640"></a>
-<span class="sourceLineNo">641</span> timestamp, new byte[]{(byte) (11 + '0')}));<a name="line.641"></a>
-<span class="sourceLineNo">642</span> log.append(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(), tableName,<a name="line.642"></a>
-<span class="sourceLineNo">643</span> System.currentTimeMillis(), mvcc, scopes), cols, true);<a name="line.643"></a>
-<span class="sourceLineNo">644</span> log.sync();<a name="line.644"></a>
-<span class="sourceLineNo">645</span> assertEquals(COL_COUNT, visitor.increments);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span><a name="line.647"></a>
-<span class="sourceLineNo">648</span> /**<a name="line.648"></a>
-<span class="sourceLineNo">649</span> * A loaded WAL coprocessor won't break existing WAL test cases.<a name="line.649"></a>
-<span class="sourceLineNo">650</span> */<a name="line.650"></a>
-<span class="sourceLineNo">651</span> @Test<a name="line.651"></a>
-<span class="sourceLineNo">652</span> public void testWALCoprocessorLoaded() throws Exception {<a name="line.652"></a>
-<span class="sourceLineNo">653</span> // test to see whether the coprocessor is loaded or not.<a name="line.653"></a>
-<span class="sourceLineNo">654</span> WALCoprocessorHost host = wals.getWAL(null).getCoprocessorHost();<a name="line.654"></a>
-<span class="sourceLineNo">655</span> Coprocessor c = host.findCoprocessor(SampleRegionWALCoprocessor.class);<a name="line.655"></a>
-<span class="sourceLineNo">656</span> assertNotNull(c);<a name="line.656"></a>
-<span class="sourceLineNo">657</span> }<a name="line.657"></a>
-<span class="sourceLineNo">658</span><a name="line.658"></a>
-<span class="sourceLineNo">659</span> static class DumbWALActionsListener implements WALActionsListener {<a name="line.659"></a>
-<span class="sourceLineNo">660</span> int increments = 0;<a name="line.660"></a>
-<span class="sourceLineNo">661</span><a name="line.661"></a>
-<span class="sourceLineNo">662</span> @Override<a name="line.662"></a>
-<span class="sourceLineNo">663</span> public void visitLogEntryBeforeWrite(RegionInfo info, WALKey logKey, WALEdit logEdit) {<a name="line.663"></a>
-<span class="sourceLineNo">664</span> increments++;<a name="line.664"></a>
-<span class="sourceLineNo">665</span> }<a name="line.665"></a>
-<span class="sourceLineNo">666</span><a name="line.666"></a>
-<span class="sourceLineNo">667</span> @Override<a name="line.667"></a>
-<span class="sourceLineNo">668</span> public void visitLogEntryBeforeWrite(WALKey logKey, WALEdit logEdit) {<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // To change body of implemented methods use File | Settings | File<a name="line.669"></a>
-<span class="sourceLineNo">670</span> // Templates.<a name="line.670"></a>
-<span class="sourceLineNo">671</span> increments++;<a name="line.671"></a>
-<span class="sourceLineNo">672</span> }<a name="line.672"></a>
-<span class="sourceLineNo">673</span> }<a name="line.673"></a>
-<span class="sourceLineNo">674</span>}<a name="line.674"></a>
+<span class="sourceLineNo">143</span> "dfs.client.block.recovery.retries", 1);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> TEST_UTIL.getConfiguration().setInt(<a name="line.144"></a>
+<span class="sourceLineNo">145</span> "hbase.ipc.client.connection.maxidletime", 500);<a name="line.145"></a>
+<span class="sourceLineNo">146</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.timeout", 10000);<a name="line.146"></a>
+<span class="sourceLineNo">147</span> TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 1000);<a name="line.147"></a>
+<span class="sourceLineNo">148</span> TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,<a name="line.148"></a>
+<span class="sourceLineNo">149</span> SampleRegionWALCoprocessor.class.getName());<a name="line.149"></a>
+<span class="sourceLineNo">150</span> TEST_UTIL.startMiniDFSCluster(3);<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span> conf = TEST_UTIL.getConfiguration();<a name="line.152"></a>
+<span class="sourceLineNo">153</span> cluster = TEST_UTIL.getDFSCluster();<a name="line.153"></a>
+<span class="sourceLineNo">154</span><a name="line.154"></a>
+<span class="sourceLineNo">155</span> hbaseDir = TEST_UTIL.createRootDir();<a name="line.155"></a>
+<span class="sourceLineNo">156</span> hbaseWALDir = TEST_UTIL.createWALRootDir();<a name="line.156"></a>
+<span class="sourceLineNo">157</span> }<a name="line.157"></a>
+<span class="sourceLineNo">158</span><a name="line.158"></a>
+<span class="sourceLineNo">159</span> @AfterClass<a name="line.159"></a>
+<span class="sourceLineNo">160</span> public static void tearDownAfterClass() throws Exception {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> TEST_UTIL.shutdownMiniCluster();<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
+<span class="sourceLineNo">163</span><a name="line.163"></a>
+<span class="sourceLineNo">164</span> @Test<a name="line.164"></a>
+<span class="sourceLineNo">165</span> public void canCloseSingleton() throws IOException {<a name="line.165"></a>
+<span class="sourceLineNo">166</span> WALFactory.getInstance(conf).close();<a name="line.166"></a>
+<span class="sourceLineNo">167</span> }<a name="line.167"></a>
+<span class="sourceLineNo">168</span><a name="line.168"></a>
+<span class="sourceLineNo">169</span> /**<a name="line.169"></a>
+<span class="sourceLineNo">170</span> * Just write multiple logs then split. Before fix for HADOOP-2283, this<a name="line.170"></a>
+<span class="sourceLineNo">171</span> * would fail.<a name="line.171"></a>
+<span class="sourceLineNo">172</span> * @throws IOException<a name="line.172"></a>
+<span class="sourceLineNo">173</span> */<a name="line.173"></a>
+<span class="sourceLineNo">174</span> @Test<a name="line.174"></a>
+<span class="sourceLineNo">175</span> public void testSplit() throws IOException {<a name="line.175"></a>
+<span class="sourceLineNo">176</span> final TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.176"></a>
+<span class="sourceLineNo">177</span> final byte [] rowName = tableName.getName();<a name="line.177"></a>
+<span class="sourceLineNo">178</span> final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.178"></a>
+<span class="sourceLineNo">179</span> final int howmany = 3;<a name="line.179"></a>
+<span class="sourceLineNo">180</span> RegionInfo[] infos = new RegionInfo[3];<a name="line.180"></a>
+<span class="sourceLineNo">181</span> Path tabledir = FSUtils.getTableDir(hbaseDir, tableName);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> fs.mkdirs(tabledir);<a name="line.182"></a>
+<span class="sourceLineNo">183</span> for (int i = 0; i < howmany; i++) {<a name="line.183"></a>
+<span class="sourceLineNo">184</span> infos[i] = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("" + i))<a name="line.184"></a>
+<span class="sourceLineNo">185</span> .setEndKey(Bytes.toBytes("" + (i + 1))).build();<a name="line.185"></a>
+<span class="sourceLineNo">186</span> fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));<a name="line.186"></a>
+<span class="sourceLineNo">187</span> LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());<a name="line.187"></a>
+<span class="sourceLineNo">188</span> }<a name="line.188"></a>
+<span class="sourceLineNo">189</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.189"></a>
+<span class="sourceLineNo">190</span> scopes.put(Bytes.toBytes("column"), 0);<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span><a name="line.192"></a>
+<span class="sourceLineNo">193</span> // Add edits for three regions.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> for (int ii = 0; ii < howmany; ii++) {<a name="line.194"></a>
+<span class="sourceLineNo">195</span> for (int i = 0; i < howmany; i++) {<a name="line.195"></a>
+<span class="sourceLineNo">196</span> final WAL log =<a name="line.196"></a>
+<span class="sourceLineNo">197</span> wals.getWAL(infos[i]);<a name="line.197"></a>
+<span class="sourceLineNo">198</span> for (int j = 0; j < howmany; j++) {<a name="line.198"></a>
+<span class="sourceLineNo">199</span> WALEdit edit = new WALEdit();<a name="line.199"></a>
+<span class="sourceLineNo">200</span> byte [] family = Bytes.toBytes("column");<a name="line.200"></a>
+<span class="sourceLineNo">201</span> byte [] qualifier = Bytes.toBytes(Integer.toString(j));<a name="line.201"></a>
+<span class="sourceLineNo">202</span> byte [] column = Bytes.toBytes("column:" + Integer.toString(j));<a name="line.202"></a>
+<span class="sourceLineNo">203</span> edit.add(new KeyValue(rowName, family, qualifier,<a name="line.203"></a>
+<span class="sourceLineNo">204</span> System.currentTimeMillis(), column));<a name="line.204"></a>
+<span class="sourceLineNo">205</span> LOG.info("Region " + i + ": " + edit);<a name="line.205"></a>
+<span class="sourceLineNo">206</span> WALKeyImpl walKey = new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName,<a name="line.206"></a>
+<span class="sourceLineNo">207</span> System.currentTimeMillis(), mvcc, scopes);<a name="line.207"></a>
+<span class="sourceLineNo">208</span> log.append(infos[i], walKey, edit, true);<a name="line.208"></a>
+<span class="sourceLineNo">209</span> walKey.getWriteEntry();<a name="line.209"></a>
+<span class="sourceLineNo">210</span> }<a name="line.210"></a>
+<span class="sourceLineNo">211</span> log.sync();<a name="line.211"></a>
+<span class="sourceLineNo">212</span> log.rollWriter(true);<a name="line.212"></a>
+<span class="sourceLineNo">213</span> }<a name="line.213"></a>
+<span class="sourceLineNo">214</span> }<a name="line.214"></a>
+<span class="sourceLineNo">215</span> wals.shutdown();<a name="line.215"></a>
+<span class="sourceLineNo">216</span> // The below calculation of logDir relies on insider information... WALSplitter should be connected better<a name="line.216"></a>
+<span class="sourceLineNo">217</span> // with the WAL system.... not requiring explicit path. The oldLogDir is just made up not used.<a name="line.217"></a>
+<span class="sourceLineNo">218</span> Path logDir =<a name="line.218"></a>
+<span class="sourceLineNo">219</span> new Path(new Path(hbaseWALDir, HConstants.HREGION_LOGDIR_NAME),<a name="line.219"></a>
+<span class="sourceLineNo">220</span> this.currentServername.toString());<a name="line.220"></a>
+<span class="sourceLineNo">221</span> Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.221"></a>
+<span class="sourceLineNo">222</span> List<Path> splits = WALSplitter.split(hbaseWALDir, logDir, oldLogDir, fs, conf, wals);<a name="line.222"></a>
+<span class="sourceLineNo">223</span> verifySplits(splits, howmany);<a name="line.223"></a>
+<span class="sourceLineNo">224</span> }<a name="line.224"></a>
+<span class="sourceLineNo">225</span><a name="line.225"></a>
+<span class="sourceLineNo">226</span> /**<a name="line.226"></a>
+<span class="sourceLineNo">227</span> * Test new HDFS-265 sync.<a name="line.227"></a>
+<span class="sourceLineNo">228</span> * @throws Exception<a name="line.228"></a>
+<span class="sourceLineNo">229</span> */<a name="line.229"></a>
+<span class="sourceLineNo">230</span> @Test<a name="line.230"></a>
+<span class="sourceLineNo">231</span> public void Broken_testSync() throws Exception {<a name="line.231"></a>
+<span class="sourceLineNo">232</span> TableName tableName = TableName.valueOf(currentTest.getMethodName());<a name="line.232"></a>
+<span class="sourceLineNo">233</span> MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);<a name="line.233"></a>
+<span class="sourceLineNo">234</span> // First verify that using streams all works.<a name="line.234"></a>
+<span class="sourceLineNo">235</span> Path p = new Path(dir, currentTest.getMethodName() + ".fsdos");<a name="line.235"></a>
+<span class="sourceLineNo">236</span> FSDataOutputStream out = fs.create(p);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> out.write(tableName.getName());<a name="line.237"></a>
+<span class="sourceLineNo">238</span> Method syncMethod = null;<a name="line.238"></a>
+<span class="sourceLineNo">239</span> try {<a name="line.239"></a>
+<span class="sourceLineNo">240</span> syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});<a name="line.240"></a>
+<span class="sourceLineNo">241</span> } catch (NoSuchMethodException e) {<a name="line.241"></a>
+<span class="sourceLineNo">242</span> try {<a name="line.242"></a>
+<span class="sourceLineNo">243</span> syncMethod = out.getClass().getMethod("sync", new Class<?> []{});<a name="line.243"></a>
+<span class="sourceLineNo">244</span> } catch (NoSuchMethodException ex) {<a name="line.244"></a>
+<span class="sourceLineNo">245</span> fail("This version of Hadoop supports neither Syncable.sync() " +<a name="line.245"></a>
+<span class="sourceLineNo">246</span> "nor Syncable.hflush().");<a name="line.246"></a>
+<span class="sourceLineNo">247</span> }<a name="line.247"></a>
+<span class="sourceLineNo">248</span> }<a name="line.248"></a>
+<span class="sourceLineNo">249</span> syncMethod.invoke(out, new Object[]{});<a name="line.249"></a>
+<span class="sourceLineNo">250</span> FSDataInputStream in = fs.open(p);<a name="line.250"></a>
+<span class="sourceLineNo">251</span> assertTrue(in.available() > 0);<a name="line.251"></a>
+<span class="sourceLineNo">252</span> byte [] buffer = new byte [1024];<a name="line.252"></a>
+<span class="sourceLineNo">253</span> int read = in.read(buffer);<a name="line.253"></a>
+<span class="sourceLineNo">254</span> assertEquals(tableName.getName().length, read);<a name="line.254"></a>
+<span class="sourceLineNo">255</span> out.close();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> in.close();<a name="line.256"></a>
+<span class="sourceLineNo">257</span><a name="line.257"></a>
+<span class="sourceLineNo">258</span> final int total = 20;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> WAL.Reader reader = null;<a name="line.259"></a>
+<span class="sourceLineNo">260</span><a name="line.260"></a>
+<span class="sourceLineNo">261</span> try {<a name="line.261"></a>
+<span class="sourceLineNo">262</span> RegionInfo info = RegionInfoBuilder.newBuilder(tableName).build();<a name="line.262"></a>
+<span class="sourceLineNo">263</span> NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> scopes.put(tableName.getName(), 0);<a name="line.264"></a>
+<span class="sourceLineNo">265</span> final WAL wal = wals.getWAL(info);<a name="line.265"></a>
+<span class="sourceLineNo">266</span><a name="line.266"></a>
+<span class="sourceLineNo">267</span> for (int i = 0; i < total; i++) {<a name="line.267"></a>
+<span class="sourceLineNo">268</span> WALEdit kvs = new WALEdit();<a name="line.268"></a>
+<span class="sourceLineNo">269</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.269"></a>
+<span class="sourceLineNo">270</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.270"></a>
+<span class="sourceLineNo">271</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> // Now call sync and try reading. Opening a Reader before you sync just<a name="line.273"></a>
+<span class="sourceLineNo">274</span> // gives you EOFE.<a name="line.274"></a>
+<span class="sourceLineNo">275</span> wal.sync();<a name="line.275"></a>
+<span class="sourceLineNo">276</span> // Open a Reader.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);<a name="line.277"></a>
+<span class="sourceLineNo">278</span> reader = wals.createReader(fs, walPath);<a name="line.278"></a>
+<span class="sourceLineNo">279</span> int count = 0;<a name="line.279"></a>
+<span class="sourceLineNo">280</span> WAL.Entry entry = new WAL.Entry();<a name="line.280"></a>
+<span class="sourceLineNo">281</span> while ((entry = reader.next(entry)) != null) count++;<a name="line.281"></a>
+<span class="sourceLineNo">282</span> assertEquals(total, count);<a name="line.282"></a>
+<span class="sourceLineNo">283</span> reader.close();<a name="line.283"></a>
+<span class="sourceLineNo">284</span> // Add test that checks to see that an open of a Reader works on a file<a name="line.284"></a>
+<span class="sourceLineNo">285</span> // that has had a sync done on it.<a name="line.285"></a>
+<span class="sourceLineNo">286</span> for (int i = 0; i < total; i++) {<a name="line.286"></a>
+<span class="sourceLineNo">287</span> WALEdit kvs = new WALEdit();<a name="line.287"></a>
+<span class="sourceLineNo">288</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));<a name="line.288"></a>
+<span class="sourceLineNo">289</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.289"></a>
+<span class="sourceLineNo">290</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.290"></a>
+<span class="sourceLineNo">291</span> }<a name="line.291"></a>
+<span class="sourceLineNo">292</span> wal.sync();<a name="line.292"></a>
+<span class="sourceLineNo">293</span> reader = wals.createReader(fs, walPath);<a name="line.293"></a>
+<span class="sourceLineNo">294</span> count = 0;<a name="line.294"></a>
+<span class="sourceLineNo">295</span> while((entry = reader.next(entry)) != null) count++;<a name="line.295"></a>
+<span class="sourceLineNo">296</span> assertTrue(count >= total);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> reader.close();<a name="line.297"></a>
+<span class="sourceLineNo">298</span> // If I sync, should see double the edits.<a name="line.298"></a>
+<span class="sourceLineNo">299</span> wal.sync();<a name="line.299"></a>
+<span class="sourceLineNo">300</span> reader = wals.createReader(fs, walPath);<a name="line.300"></a>
+<span class="sourceLineNo">301</span> count = 0;<a name="line.301"></a>
+<span class="sourceLineNo">302</span> while((entry = reader.next(entry)) != null) count++;<a name="line.302"></a>
+<span class="sourceLineNo">303</span> assertEquals(total * 2, count);<a name="line.303"></a>
+<span class="sourceLineNo">304</span> reader.close();<a name="line.304"></a>
+<span class="sourceLineNo">305</span> // Now do a test that ensures stuff works when we go over block boundary,<a name="line.305"></a>
+<span class="sourceLineNo">306</span> // especially that we return good length on file.<a name="line.306"></a>
+<span class="sourceLineNo">307</span> final byte [] value = new byte[1025 * 1024]; // Make a 1M value.<a name="line.307"></a>
+<span class="sourceLineNo">308</span> for (int i = 0; i < total; i++) {<a name="line.308"></a>
+<span class="sourceLineNo">309</span> WALEdit kvs = new WALEdit();<a name="line.309"></a>
+<span class="sourceLineNo">310</span> kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value));<a name="line.310"></a>
+<span class="sourceLineNo">311</span> wal.append(info, new WALKeyImpl(info.getEncodedNameAsBytes(), tableName,<a name="line.311"></a>
+<span class="sourceLineNo">312</span> System.currentTimeMillis(), mvcc, scopes), kvs, true);<a name="line.312"></a>
+<span class="sourceLineNo">313</span> }<a name="line.313"></a>
+<span class="sourceLineNo">314</span> // Now I should have written out lots of blocks. Sync then read.<a name="line.314"></a>
+<span class="sourceLineNo">315</span> wal.sync();<a name="line.315"></a>
+<span class="sourceLineNo">316</span> reader = wals.createReader(fs, walPath);<a name="line.316"></a>
+<span class="sourceLineNo">317</span> count = 0;<a name="line.317"></a>
+<span class="sourceLineNo">318</span> while((entry = reader.next(entry)) != null) count++;<a name="line.318"></a>
+<span class="sourceLineNo">319</span> assertEquals(total * 3, count);<a name="line.319"></a>
+<span class="sourceLineNo">320</span> reader.close();<a name="line.320"></a>
+<span class="sourceLineNo">321</span> // shutdown and ensure that Reader gets right length also.<a name="line.321"></a>
+<span class="sourceLineNo">322</span> wal.shutdown();<a name="line.322"></a>
+<span class="sourceLineNo">323</span> reader = wals.createReader(fs, walPath);<a name="line.323"></a>
+<span class="sourceLineNo">324</span> count = 0;<a name="line.324"></a>
+<span class="sourceLineNo">325</span> while((entry = reader.next(entry)) != null) count++;<a name="line.325"></a>
+<span class="sourceLineNo">326</span> assertEquals(total * 3, count);<a name="line.326"></a>
+<span class="sourceLineNo">327</span> reader.close();<a name="line.327"></a>
+<span class="sourceLineNo">328</span> } fi
<TRUNCATED>
[04/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/downloads.html
----------------------------------------------------------------------
diff --git a/downloads.html b/downloads.html
index 8979bea..b86726d 100644
--- a/downloads.html
+++ b/downloads.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Apache HBase Downloads</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -366,7 +366,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/export_control.html
----------------------------------------------------------------------
diff --git a/export_control.html b/export_control.html
index 452535e..6a16070 100644
--- a/export_control.html
+++ b/export_control.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Export Control
@@ -331,7 +331,7 @@ for more details.</p>
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 0195bc9..6b39910 100644
--- a/index.html
+++ b/index.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Apache HBase™ Home</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -411,7 +411,7 @@ Apache HBase is an open-source, distributed, versioned, non-relational database
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/integration.html
----------------------------------------------------------------------
diff --git a/integration.html b/integration.html
index bc63fe6..cf808bf 100644
--- a/integration.html
+++ b/integration.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – CI Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -291,7 +291,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/issue-tracking.html
----------------------------------------------------------------------
diff --git a/issue-tracking.html b/issue-tracking.html
index 0ce809d..639cb70 100644
--- a/issue-tracking.html
+++ b/issue-tracking.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Issue Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -288,7 +288,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/license.html
----------------------------------------------------------------------
diff --git a/license.html b/license.html
index 0485a2b..37e3114 100644
--- a/license.html
+++ b/license.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Licenses</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -491,7 +491,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/mail-lists.html
----------------------------------------------------------------------
diff --git a/mail-lists.html b/mail-lists.html
index 5549fad..75ac1c2 100644
--- a/mail-lists.html
+++ b/mail-lists.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Mailing Lists</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -341,7 +341,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/metrics.html
----------------------------------------------------------------------
diff --git a/metrics.html b/metrics.html
index c180dce..457149d 100644
--- a/metrics.html
+++ b/metrics.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Apache HBase (TM) Metrics
@@ -459,7 +459,7 @@ export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxrem
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/old_news.html
----------------------------------------------------------------------
diff --git a/old_news.html b/old_news.html
index 318c6dc..8eb375d 100644
--- a/old_news.html
+++ b/old_news.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Old Apache HBase (TM) News
@@ -440,7 +440,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/plugin-management.html
----------------------------------------------------------------------
diff --git a/plugin-management.html b/plugin-management.html
index 7ac1356..bef526b 100644
--- a/plugin-management.html
+++ b/plugin-management.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Plugin Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -440,7 +440,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/plugins.html
----------------------------------------------------------------------
diff --git a/plugins.html b/plugins.html
index edaf742..c56cba3 100644
--- a/plugins.html
+++ b/plugins.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Plugins</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -375,7 +375,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/poweredbyhbase.html
----------------------------------------------------------------------
diff --git a/poweredbyhbase.html b/poweredbyhbase.html
index 0d2f69f..f7dcd67 100644
--- a/poweredbyhbase.html
+++ b/poweredbyhbase.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Powered By Apache HBase</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -769,7 +769,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/project-info.html
----------------------------------------------------------------------
diff --git a/project-info.html b/project-info.html
index 37e3d20..2311a10 100644
--- a/project-info.html
+++ b/project-info.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Information</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -335,7 +335,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/project-reports.html
----------------------------------------------------------------------
diff --git a/project-reports.html b/project-reports.html
index b59fea6..e98674e 100644
--- a/project-reports.html
+++ b/project-reports.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Generated Reports</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -305,7 +305,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/project-summary.html
----------------------------------------------------------------------
diff --git a/project-summary.html b/project-summary.html
index 0cddb73..53a0932 100644
--- a/project-summary.html
+++ b/project-summary.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Summary</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -331,7 +331,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/pseudo-distributed.html
----------------------------------------------------------------------
diff --git a/pseudo-distributed.html b/pseudo-distributed.html
index fadd7d5..e0e395b 100644
--- a/pseudo-distributed.html
+++ b/pseudo-distributed.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Running Apache HBase (TM) in pseudo-distributed mode
@@ -308,7 +308,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/replication.html
----------------------------------------------------------------------
diff --git a/replication.html b/replication.html
index d39786a..3068561 100644
--- a/replication.html
+++ b/replication.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Apache HBase (TM) Replication
@@ -303,7 +303,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/resources.html
----------------------------------------------------------------------
diff --git a/resources.html b/resources.html
index ab5ce10..debdeb6 100644
--- a/resources.html
+++ b/resources.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Other Apache HBase (TM) Resources</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -331,7 +331,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/source-repository.html
----------------------------------------------------------------------
diff --git a/source-repository.html b/source-repository.html
index b0fb0f6..7fed5f3 100644
--- a/source-repository.html
+++ b/source-repository.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Source Code Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -299,7 +299,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/sponsors.html
----------------------------------------------------------------------
diff --git a/sponsors.html b/sponsors.html
index cc82af3..fb7ec75 100644
--- a/sponsors.html
+++ b/sponsors.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Apache HBase Sponsors</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -333,7 +333,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/supportingprojects.html
----------------------------------------------------------------------
diff --git a/supportingprojects.html b/supportingprojects.html
index 2127569..224c465 100644
--- a/supportingprojects.html
+++ b/supportingprojects.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Supporting Projects</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -520,7 +520,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/team-list.html
----------------------------------------------------------------------
diff --git a/team-list.html b/team-list.html
index 3c8e71e..b31c530 100644
--- a/team-list.html
+++ b/team-list.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180615" />
+ <meta name="Date-Revision-yyyymmdd" content="20180616" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Team</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.5-HBASE.min.css" />
@@ -736,7 +736,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-06-15</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-06-16</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html b/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
index 1ea54f3..10b8c3c 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.659">TestWALFactory.DumbWALActionsListener</a>
+<pre>static class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.661">TestWALFactory.DumbWALActionsListener</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
implements org.apache.hadoop.hbase.regionserver.wal.WALActionsListener</pre>
</li>
@@ -220,7 +220,7 @@ implements org.apache.hadoop.hbase.regionserver.wal.WALActionsListener</pre>
<ul class="blockListLast">
<li class="blockList">
<h4>increments</h4>
-<pre>int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.660">increments</a></pre>
+<pre>int <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.662">increments</a></pre>
</li>
</ul>
</li>
@@ -237,7 +237,7 @@ implements org.apache.hadoop.hbase.regionserver.wal.WALActionsListener</pre>
<ul class="blockListLast">
<li class="blockList">
<h4>DumbWALActionsListener</h4>
-<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.659">DumbWALActionsListener</a>()</pre>
+<pre><a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.661">DumbWALActionsListener</a>()</pre>
</li>
</ul>
</li>
@@ -254,7 +254,7 @@ implements org.apache.hadoop.hbase.regionserver.wal.WALActionsListener</pre>
<ul class="blockList">
<li class="blockList">
<h4>visitLogEntryBeforeWrite</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.663">visitLogEntryBeforeWrite</a>(org.apache.hadoop.hbase.client.RegionInfo info,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.665">visitLogEntryBeforeWrite</a>(org.apache.hadoop.hbase.client.RegionInfo info,
org.apache.hadoop.hbase.wal.WALKey logKey,
org.apache.hadoop.hbase.wal.WALEdit logEdit)</pre>
<dl>
@@ -269,7 +269,7 @@ implements org.apache.hadoop.hbase.regionserver.wal.WALActionsListener</pre>
<ul class="blockListLast">
<li class="blockList">
<h4>visitLogEntryBeforeWrite</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.668">visitLogEntryBeforeWrite</a>(org.apache.hadoop.hbase.wal.WALKey logKey,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.DumbWALActionsListener.html#line.670">visitLogEntryBeforeWrite</a>(org.apache.hadoop.hbase.wal.WALKey logKey,
org.apache.hadoop.hbase.wal.WALEdit logEdit)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.html b/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.html
index c0c19c9..e924a67 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/wal/TestWALFactory.html
@@ -109,7 +109,7 @@ var activeTableTab = "activeTableTab";
<li class="blockList">
<hr>
<br>
-<pre>public class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.81">TestWALFactory</a>
+<pre>public class <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.82">TestWALFactory</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">WAL tests that can be reused across providers.</div>
</li>
@@ -320,7 +320,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>CLASS_RULE</h4>
-<pre>public static final <a href="../../../../../org/apache/hadoop/hbase/HBaseClassTestRule.html" title="class in org.apache.hadoop.hbase">HBaseClassTestRule</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.84">CLASS_RULE</a></pre>
+<pre>public static final <a href="../../../../../org/apache/hadoop/hbase/HBaseClassTestRule.html" title="class in org.apache.hadoop.hbase">HBaseClassTestRule</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.85">CLASS_RULE</a></pre>
</li>
</ul>
<a name="LOG">
@@ -329,7 +329,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>LOG</h4>
-<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.87">LOG</a></pre>
+<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.88">LOG</a></pre>
</li>
</ul>
<a name="conf">
@@ -338,7 +338,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>conf</h4>
-<pre>protected static org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.89">conf</a></pre>
+<pre>protected static org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.90">conf</a></pre>
</li>
</ul>
<a name="cluster">
@@ -347,7 +347,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>cluster</h4>
-<pre>private static org.apache.hadoop.hdfs.MiniDFSCluster <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.90">cluster</a></pre>
+<pre>private static org.apache.hadoop.hdfs.MiniDFSCluster <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.91">cluster</a></pre>
</li>
</ul>
<a name="TEST_UTIL">
@@ -356,7 +356,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>TEST_UTIL</h4>
-<pre>protected static final <a href="../../../../../org/apache/hadoop/hbase/HBaseTestingUtility.html" title="class in org.apache.hadoop.hbase">HBaseTestingUtility</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.91">TEST_UTIL</a></pre>
+<pre>protected static final <a href="../../../../../org/apache/hadoop/hbase/HBaseTestingUtility.html" title="class in org.apache.hadoop.hbase">HBaseTestingUtility</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.92">TEST_UTIL</a></pre>
</li>
</ul>
<a name="hbaseDir">
@@ -365,7 +365,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>hbaseDir</h4>
-<pre>protected static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.92">hbaseDir</a></pre>
+<pre>protected static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.93">hbaseDir</a></pre>
</li>
</ul>
<a name="hbaseWALDir">
@@ -374,7 +374,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>hbaseWALDir</h4>
-<pre>protected static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.93">hbaseWALDir</a></pre>
+<pre>protected static org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.94">hbaseWALDir</a></pre>
</li>
</ul>
<a name="fs">
@@ -383,7 +383,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>fs</h4>
-<pre>protected org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.95">fs</a></pre>
+<pre>protected org.apache.hadoop.fs.FileSystem <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.96">fs</a></pre>
</li>
</ul>
<a name="dir">
@@ -392,7 +392,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>dir</h4>
-<pre>protected org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.96">dir</a></pre>
+<pre>protected org.apache.hadoop.fs.Path <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.97">dir</a></pre>
</li>
</ul>
<a name="wals">
@@ -401,7 +401,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>wals</h4>
-<pre>protected org.apache.hadoop.hbase.wal.WALFactory <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.97">wals</a></pre>
+<pre>protected org.apache.hadoop.hbase.wal.WALFactory <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.98">wals</a></pre>
</li>
</ul>
<a name="currentServername">
@@ -410,7 +410,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>currentServername</h4>
-<pre>private org.apache.hadoop.hbase.ServerName <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.98">currentServername</a></pre>
+<pre>private org.apache.hadoop.hbase.ServerName <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.99">currentServername</a></pre>
</li>
</ul>
<a name="currentTest">
@@ -419,7 +419,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>currentTest</h4>
-<pre>public final org.junit.rules.TestName <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.101">currentTest</a></pre>
+<pre>public final org.junit.rules.TestName <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.102">currentTest</a></pre>
</li>
</ul>
</li>
@@ -436,7 +436,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>TestWALFactory</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.81">TestWALFactory</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.82">TestWALFactory</a>()</pre>
</li>
</ul>
</li>
@@ -453,7 +453,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setUp</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.104">setUp</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.105">setUp</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -467,7 +467,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>tearDown</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.112">tearDown</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.113">tearDown</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -481,7 +481,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setUpBeforeClass</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.128">setUpBeforeClass</a>()
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.129">setUpBeforeClass</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -495,7 +495,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>tearDownAfterClass</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.158">tearDownAfterClass</a>()
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.160">tearDownAfterClass</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -509,7 +509,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>canCloseSingleton</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.163">canCloseSingleton</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.165">canCloseSingleton</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -523,7 +523,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testSplit</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.173">testSplit</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.175">testSplit</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Just write multiple logs then split. Before fix for HADOOP-2283, this
would fail.</div>
@@ -539,7 +539,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>Broken_testSync</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.229">Broken_testSync</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.231">Broken_testSync</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<div class="block">Test new HDFS-265 sync.</div>
<dl>
@@ -554,7 +554,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>verifySplits</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.331">verifySplits</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> splits,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.333">verifySplits</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><org.apache.hadoop.fs.Path> splits,
int howmany)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -569,7 +569,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testAppendClose</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.372">testAppendClose</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.374">testAppendClose</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -583,7 +583,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testEditAdd</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.493">testEditAdd</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.495">testEditAdd</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Tests that we can write out an edit, close, and then read it back in again.</div>
<dl>
@@ -598,7 +598,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testAppend</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.554">testAppend</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.556">testAppend</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -612,7 +612,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testVisitors</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.614">testVisitors</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.616">testVisitors</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<div class="block">Test that we can visit entries before they are appended</div>
<dl>
@@ -627,7 +627,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>testWALCoprocessorLoaded</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.652">testWALCoprocessorLoaded</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/wal/TestWALFactory.html#line.654">testWALCoprocessorLoaded</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<div class="block">A loaded WAL coprocessor won't break existing WAL test cases.</div>
<dl>
[08/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.SinkWriter.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span> }<a name="line.68
<TRUNCATED>
[09/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.RegionEntryBuffer.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo
<TRUNCATED>
[05/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span> }<a name="line.682"></a>
+<span class="sourceLineNo">683</span> }<a n
<TRUNCATED>
[16/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.BoundedLogWriterCreationOutputSink.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all r
<TRUNCATED>
[10/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.PipelineController.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceL
<TRUNCATED>
[07/22] hbase-site git commit: Published site at
ac5bb8155b618194fe9cf1131f0e72c99b7b534c.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/a7f7dba1/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
index 99f53c4..2d5c946 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/wal/WALSplitter.WriterAndPath.html
@@ -123,7 +123,7 @@
<span class="sourceLineNo">115</span> public static final boolean SPLIT_SKIP_ERRORS_DEFAULT = false;<a name="line.115"></a>
<span class="sourceLineNo">116</span><a name="line.116"></a>
<span class="sourceLineNo">117</span> // Parameters for split process<a name="line.117"></a>
-<span class="sourceLineNo">118</span> protected final Path rootDir;<a name="line.118"></a>
+<span class="sourceLineNo">118</span> protected final Path walDir;<a name="line.118"></a>
<span class="sourceLineNo">119</span> protected final FileSystem fs;<a name="line.119"></a>
<span class="sourceLineNo">120</span> protected final Configuration conf;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
@@ -156,14 +156,14 @@
<span class="sourceLineNo">148</span><a name="line.148"></a>
<span class="sourceLineNo">149</span><a name="line.149"></a>
<span class="sourceLineNo">150</span> @VisibleForTesting<a name="line.150"></a>
-<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path rootDir,<a name="line.151"></a>
+<span class="sourceLineNo">151</span> WALSplitter(final WALFactory factory, Configuration conf, Path walDir,<a name="line.151"></a>
<span class="sourceLineNo">152</span> FileSystem fs, LastSequenceId idChecker,<a name="line.152"></a>
<span class="sourceLineNo">153</span> SplitLogWorkerCoordination splitLogWorkerCoordination) {<a name="line.153"></a>
<span class="sourceLineNo">154</span> this.conf = HBaseConfiguration.create(conf);<a name="line.154"></a>
<span class="sourceLineNo">155</span> String codecClassName = conf<a name="line.155"></a>
<span class="sourceLineNo">156</span> .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());<a name="line.156"></a>
<span class="sourceLineNo">157</span> this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> this.rootDir = rootDir;<a name="line.158"></a>
+<span class="sourceLineNo">158</span> this.walDir = walDir;<a name="line.158"></a>
<span class="sourceLineNo">159</span> this.fs = fs;<a name="line.159"></a>
<span class="sourceLineNo">160</span> this.sequenceIdChecker = idChecker;<a name="line.160"></a>
<span class="sourceLineNo">161</span> this.splitLogWorkerCoordination = splitLogWorkerCoordination;<a name="line.161"></a>
@@ -194,11 +194,11 @@
<span class="sourceLineNo">186</span> * <p><a name="line.186"></a>
<span class="sourceLineNo">187</span> * @return false if it is interrupted by the progress-able.<a name="line.187"></a>
<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path rootDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
+<span class="sourceLineNo">189</span> public static boolean splitLogFile(Path walDir, FileStatus logfile, FileSystem fs,<a name="line.189"></a>
<span class="sourceLineNo">190</span> Configuration conf, CancelableProgressable reporter, LastSequenceId idChecker,<a name="line.190"></a>
<span class="sourceLineNo">191</span> SplitLogWorkerCoordination splitLogWorkerCoordination, final WALFactory factory)<a name="line.191"></a>
<span class="sourceLineNo">192</span> throws IOException {<a name="line.192"></a>
-<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, rootDir, fs, idChecker,<a name="line.193"></a>
+<span class="sourceLineNo">193</span> WALSplitter s = new WALSplitter(factory, conf, walDir, fs, idChecker,<a name="line.193"></a>
<span class="sourceLineNo">194</span> splitLogWorkerCoordination);<a name="line.194"></a>
<span class="sourceLineNo">195</span> return s.splitLogFile(logfile, reporter);<a name="line.195"></a>
<span class="sourceLineNo">196</span> }<a name="line.196"></a>
@@ -330,10 +330,10 @@
<span class="sourceLineNo">322</span> LOG.warn("Could not parse, corrupted WAL={}", logPath, e);<a name="line.322"></a>
<span class="sourceLineNo">323</span> if (splitLogWorkerCoordination != null) {<a name="line.323"></a>
<span class="sourceLineNo">324</span> // Some tests pass in a csm of null.<a name="line.324"></a>
-<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
+<span class="sourceLineNo">325</span> splitLogWorkerCoordination.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.325"></a>
<span class="sourceLineNo">326</span> } else {<a name="line.326"></a>
<span class="sourceLineNo">327</span> // for tests only<a name="line.327"></a>
-<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
+<span class="sourceLineNo">328</span> ZKSplitLog.markCorrupted(walDir, logfile.getPath().getName(), fs);<a name="line.328"></a>
<span class="sourceLineNo">329</span> }<a name="line.329"></a>
<span class="sourceLineNo">330</span> isCorrupted = true;<a name="line.330"></a>
<span class="sourceLineNo">331</span> } catch (IOException e) {<a name="line.331"></a>
@@ -465,1437 +465,1442 @@
<span class="sourceLineNo">457</span> * <code>logEntry</code>: e.g. /hbase/some_table/2323432434/recovered.edits/2332.<a name="line.457"></a>
<span class="sourceLineNo">458</span> * This method also ensures existence of RECOVERED_EDITS_DIR under the region<a name="line.458"></a>
<span class="sourceLineNo">459</span> * creating it if necessary.<a name="line.459"></a>
-<span class="sourceLineNo">460</span> * @param fs<a name="line.460"></a>
-<span class="sourceLineNo">461</span> * @param logEntry<a name="line.461"></a>
-<span class="sourceLineNo">462</span> * @param rootDir HBase root dir.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.463"></a>
-<span class="sourceLineNo">464</span> * @return Path to file into which to dump split log edits.<a name="line.464"></a>
-<span class="sourceLineNo">465</span> * @throws IOException<a name="line.465"></a>
-<span class="sourceLineNo">466</span> */<a name="line.466"></a>
-<span class="sourceLineNo">467</span> @SuppressWarnings("deprecation")<a name="line.467"></a>
-<span class="sourceLineNo">468</span> @VisibleForTesting<a name="line.468"></a>
-<span class="sourceLineNo">469</span> static Path getRegionSplitEditsPath(final FileSystem fs,<a name="line.469"></a>
-<span class="sourceLineNo">470</span> final Entry logEntry, final Path rootDir, String fileNameBeingSplit)<a name="line.470"></a>
-<span class="sourceLineNo">471</span> throws IOException {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.473"></a>
-<span class="sourceLineNo">474</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.475"></a>
-<span class="sourceLineNo">476</span><a name="line.476"></a>
-<span class="sourceLineNo">477</span> if (!fs.exists(regiondir)) {<a name="line.477"></a>
-<span class="sourceLineNo">478</span> LOG.info("This region's directory does not exist: {}."<a name="line.478"></a>
-<span class="sourceLineNo">479</span> + "It is very likely that it was already split so it is "<a name="line.479"></a>
-<span class="sourceLineNo">480</span> + "safe to discard those edits.", regiondir);<a name="line.480"></a>
-<span class="sourceLineNo">481</span> return null;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> }<a name="line.482"></a>
-<span class="sourceLineNo">483</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> Path tmp = new Path("/tmp");<a name="line.484"></a>
-<span class="sourceLineNo">485</span> if (!fs.exists(tmp)) {<a name="line.485"></a>
-<span class="sourceLineNo">486</span> fs.mkdirs(tmp);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> }<a name="line.487"></a>
-<span class="sourceLineNo">488</span> tmp = new Path(tmp,<a name="line.488"></a>
-<span class="sourceLineNo">489</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.490"></a>
-<span class="sourceLineNo">491</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.491"></a>
-<span class="sourceLineNo">492</span> + "So moving it to {}", dir, tmp);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> if (!fs.rename(dir, tmp)) {<a name="line.493"></a>
-<span class="sourceLineNo">494</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> LOG.warn("mkdir failed on {}", dir);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.501"></a>
-<span class="sourceLineNo">502</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // region's replayRecoveredEdits will not delete it<a name="line.503"></a>
-<span class="sourceLineNo">504</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.504"></a>
-<span class="sourceLineNo">505</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.505"></a>
-<span class="sourceLineNo">506</span> return new Path(dir, fileName);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span><a name="line.508"></a>
-<span class="sourceLineNo">509</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> /**<a name="line.513"></a>
-<span class="sourceLineNo">514</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.514"></a>
-<span class="sourceLineNo">515</span> * in the file from its first edit. Then we could use the name to skip<a name="line.515"></a>
-<span class="sourceLineNo">516</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.516"></a>
-<span class="sourceLineNo">517</span> * @param srcPath<a name="line.517"></a>
-<span class="sourceLineNo">518</span> * @param maximumEditLogSeqNum<a name="line.518"></a>
-<span class="sourceLineNo">519</span> * @return dstPath take file's last edit log seq num as the name<a name="line.519"></a>
-<span class="sourceLineNo">520</span> */<a name="line.520"></a>
-<span class="sourceLineNo">521</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.521"></a>
-<span class="sourceLineNo">522</span> long maximumEditLogSeqNum) {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return new Path(srcPath.getParent(), fileName);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @VisibleForTesting<a name="line.527"></a>
-<span class="sourceLineNo">528</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> return String.format("%019d", seqid);<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.532"></a>
-<span class="sourceLineNo">533</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.533"></a>
-<span class="sourceLineNo">534</span><a name="line.534"></a>
-<span class="sourceLineNo">535</span> /**<a name="line.535"></a>
-<span class="sourceLineNo">536</span> * @param regiondir<a name="line.536"></a>
-<span class="sourceLineNo">537</span> * This regions directory in the filesystem.<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * @return The directory that holds recovered edits files for the region<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * <code>regiondir</code><a name="line.539"></a>
-<span class="sourceLineNo">540</span> */<a name="line.540"></a>
-<span class="sourceLineNo">541</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span><a name="line.544"></a>
-<span class="sourceLineNo">545</span> /**<a name="line.545"></a>
-<span class="sourceLineNo">546</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.546"></a>
-<span class="sourceLineNo">547</span> * with '.temp' suffix.<a name="line.547"></a>
-<span class="sourceLineNo">548</span> *<a name="line.548"></a>
-<span class="sourceLineNo">549</span> * @param fs<a name="line.549"></a>
-<span class="sourceLineNo">550</span> * @param regiondir<a name="line.550"></a>
-<span class="sourceLineNo">551</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.551"></a>
-<span class="sourceLineNo">552</span> * @throws IOException<a name="line.552"></a>
-<span class="sourceLineNo">553</span> */<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.554"></a>
-<span class="sourceLineNo">555</span> final Path regiondir) throws IOException {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> if (!fs.exists(editsdir)) {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> return filesSorted;<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.561"></a>
-<span class="sourceLineNo">562</span> @Override<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public boolean accept(Path p) {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> boolean result = false;<a name="line.564"></a>
-<span class="sourceLineNo">565</span> try {<a name="line.565"></a>
-<span class="sourceLineNo">566</span> // Return files and only files that match the editfile names pattern.<a name="line.566"></a>
-<span class="sourceLineNo">567</span> // There can be other files in this directory other than edit files.<a name="line.567"></a>
-<span class="sourceLineNo">568</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.568"></a>
-<span class="sourceLineNo">569</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.569"></a>
-<span class="sourceLineNo">570</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.570"></a>
-<span class="sourceLineNo">571</span> result = fs.isFile(p) && m.matches();<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // because it means splitwal thread is writting this file.<a name="line.573"></a>
-<span class="sourceLineNo">574</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.574"></a>
-<span class="sourceLineNo">575</span> result = false;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> }<a name="line.576"></a>
-<span class="sourceLineNo">577</span> // Skip SeqId Files<a name="line.577"></a>
-<span class="sourceLineNo">578</span> if (isSequenceIdFile(p)) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> result = false;<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span> } catch (IOException e) {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.582"></a>
-<span class="sourceLineNo">583</span> }<a name="line.583"></a>
-<span class="sourceLineNo">584</span> return result;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> }<a name="line.585"></a>
-<span class="sourceLineNo">586</span> });<a name="line.586"></a>
-<span class="sourceLineNo">587</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.587"></a>
-<span class="sourceLineNo">588</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.588"></a>
-<span class="sourceLineNo">589</span> }<a name="line.589"></a>
-<span class="sourceLineNo">590</span> return filesSorted;<a name="line.590"></a>
-<span class="sourceLineNo">591</span> }<a name="line.591"></a>
-<span class="sourceLineNo">592</span><a name="line.592"></a>
-<span class="sourceLineNo">593</span> /**<a name="line.593"></a>
-<span class="sourceLineNo">594</span> * Move aside a bad edits file.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> *<a name="line.595"></a>
-<span class="sourceLineNo">596</span> * @param fs<a name="line.596"></a>
-<span class="sourceLineNo">597</span> * @param edits<a name="line.597"></a>
-<span class="sourceLineNo">598</span> * Edits file to move aside.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> * @return The name of the moved aside file.<a name="line.599"></a>
-<span class="sourceLineNo">600</span> * @throws IOException<a name="line.600"></a>
-<span class="sourceLineNo">601</span> */<a name="line.601"></a>
-<span class="sourceLineNo">602</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.602"></a>
-<span class="sourceLineNo">603</span> throws IOException {<a name="line.603"></a>
-<span class="sourceLineNo">604</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.604"></a>
-<span class="sourceLineNo">605</span> + System.currentTimeMillis());<a name="line.605"></a>
-<span class="sourceLineNo">606</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.606"></a>
-<span class="sourceLineNo">607</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.607"></a>
-<span class="sourceLineNo">608</span> }<a name="line.608"></a>
-<span class="sourceLineNo">609</span> return moveAsideName;<a name="line.609"></a>
-<span class="sourceLineNo">610</span> }<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.612"></a>
-<span class="sourceLineNo">613</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.613"></a>
-<span class="sourceLineNo">614</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.614"></a>
-<span class="sourceLineNo">615</span><a name="line.615"></a>
-<span class="sourceLineNo">616</span> /**<a name="line.616"></a>
-<span class="sourceLineNo">617</span> * Is the given file a region open sequence id file.<a name="line.617"></a>
-<span class="sourceLineNo">618</span> */<a name="line.618"></a>
-<span class="sourceLineNo">619</span> @VisibleForTesting<a name="line.619"></a>
-<span class="sourceLineNo">620</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.620"></a>
-<span class="sourceLineNo">621</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.621"></a>
-<span class="sourceLineNo">622</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.622"></a>
-<span class="sourceLineNo">623</span> }<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.626"></a>
-<span class="sourceLineNo">627</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.627"></a>
-<span class="sourceLineNo">628</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.628"></a>
-<span class="sourceLineNo">629</span> try {<a name="line.629"></a>
-<span class="sourceLineNo">630</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.630"></a>
-<span class="sourceLineNo">631</span> return files != null ? files : new FileStatus[0];<a name="line.631"></a>
-<span class="sourceLineNo">632</span> } catch (FileNotFoundException e) {<a name="line.632"></a>
-<span class="sourceLineNo">633</span> return new FileStatus[0];<a name="line.633"></a>
-<span class="sourceLineNo">634</span> }<a name="line.634"></a>
-<span class="sourceLineNo">635</span> }<a name="line.635"></a>
-<span class="sourceLineNo">636</span><a name="line.636"></a>
-<span class="sourceLineNo">637</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.637"></a>
-<span class="sourceLineNo">638</span> long maxSeqId = -1L;<a name="line.638"></a>
-<span class="sourceLineNo">639</span> for (FileStatus file : files) {<a name="line.639"></a>
-<span class="sourceLineNo">640</span> String fileName = file.getPath().getName();<a name="line.640"></a>
-<span class="sourceLineNo">641</span> try {<a name="line.641"></a>
-<span class="sourceLineNo">642</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.642"></a>
-<span class="sourceLineNo">643</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.643"></a>
-<span class="sourceLineNo">644</span> } catch (NumberFormatException ex) {<a name="line.644"></a>
-<span class="sourceLineNo">645</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.645"></a>
-<span class="sourceLineNo">646</span> }<a name="line.646"></a>
-<span class="sourceLineNo">647</span> }<a name="line.647"></a>
-<span class="sourceLineNo">648</span> return maxSeqId;<a name="line.648"></a>
-<span class="sourceLineNo">649</span> }<a name="line.649"></a>
-<span class="sourceLineNo">650</span><a name="line.650"></a>
-<span class="sourceLineNo">651</span> /**<a name="line.651"></a>
-<span class="sourceLineNo">652</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.652"></a>
-<span class="sourceLineNo">653</span> */<a name="line.653"></a>
-<span class="sourceLineNo">654</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.654"></a>
-<span class="sourceLineNo">655</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.655"></a>
-<span class="sourceLineNo">656</span> }<a name="line.656"></a>
-<span class="sourceLineNo">657</span><a name="line.657"></a>
-<span class="sourceLineNo">658</span> /**<a name="line.658"></a>
-<span class="sourceLineNo">659</span> * Create a file with name as region's max sequence id<a name="line.659"></a>
-<span class="sourceLineNo">660</span> */<a name="line.660"></a>
-<span class="sourceLineNo">661</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.661"></a>
-<span class="sourceLineNo">662</span> throws IOException {<a name="line.662"></a>
-<span class="sourceLineNo">663</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.663"></a>
-<span class="sourceLineNo">664</span> long maxSeqId = getMaxSequenceId(files);<a name="line.664"></a>
-<span class="sourceLineNo">665</span> if (maxSeqId > newMaxSeqId) {<a name="line.665"></a>
-<span class="sourceLineNo">666</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.666"></a>
-<span class="sourceLineNo">667</span> " is less than the old max sequence id " + maxSeqId);<a name="line.667"></a>
-<span class="sourceLineNo">668</span> }<a name="line.668"></a>
-<span class="sourceLineNo">669</span> // write a new seqId file<a name="line.669"></a>
-<span class="sourceLineNo">670</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.670"></a>
-<span class="sourceLineNo">671</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.671"></a>
-<span class="sourceLineNo">672</span> if (newMaxSeqId != maxSeqId) {<a name="line.672"></a>
-<span class="sourceLineNo">673</span> try {<a name="line.673"></a>
-<span class="sourceLineNo">674</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.674"></a>
-<span class="sourceLineNo">675</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.675"></a>
-<span class="sourceLineNo">676</span> }<a name="line.676"></a>
-<span class="sourceLineNo">677</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> maxSeqId);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> } catch (FileAlreadyExistsException ignored) {<a name="line.679"></a>
-<span class="sourceLineNo">680</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.680"></a>
-<span class="sourceLineNo">681</span> }<a name="line.681"></a>
-<span class="sourceLineNo">682</span> }<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // remove old ones<a name="line.683"></a>
-<span class="sourceLineNo">684</span> for (FileStatus status : files) {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> if (!newSeqIdFile.equals(status.getPath())) {<a name="line.685"></a>
-<span class="sourceLineNo">686</span> fs.delete(status.getPath(), false);<a name="line.686"></a>
-<span class="sourceLineNo">687</span> }<a name="line.687"></a>
-<span class="sourceLineNo">688</span> }<a name="line.688"></a>
-<span class="sourceLineNo">689</span> }<a name="line.689"></a>
-<span class="sourceLineNo">690</span><a name="line.690"></a>
-<span class="sourceLineNo">691</span> /**<a name="line.691"></a>
-<span class="sourceLineNo">692</span> * Create a new {@link Reader} for reading logs to split.<a name="line.692"></a>
-<span class="sourceLineNo">693</span> *<a name="line.693"></a>
-<span class="sourceLineNo">694</span> * @param file<a name="line.694"></a>
-<span class="sourceLineNo">695</span> * @return A new Reader instance, caller should close<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @throws IOException<a name="line.696"></a>
-<span class="sourceLineNo">697</span> * @throws CorruptedLogFileException<a name="line.697"></a>
-<span class="sourceLineNo">698</span> */<a name="line.698"></a>
-<span class="sourceLineNo">699</span> protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter)<a name="line.699"></a>
-<span class="sourceLineNo">700</span> throws IOException, CorruptedLogFileException {<a name="line.700"></a>
-<span class="sourceLineNo">701</span> Path path = file.getPath();<a name="line.701"></a>
-<span class="sourceLineNo">702</span> long length = file.getLen();<a name="line.702"></a>
-<span class="sourceLineNo">703</span> Reader in;<a name="line.703"></a>
-<span class="sourceLineNo">704</span><a name="line.704"></a>
-<span class="sourceLineNo">705</span> // Check for possibly empty file. With appends, currently Hadoop reports a<a name="line.705"></a>
-<span class="sourceLineNo">706</span> // zero length even if the file has been sync'd. Revisit if HDFS-376 or<a name="line.706"></a>
-<span class="sourceLineNo">707</span> // HDFS-878 is committed.<a name="line.707"></a>
-<span class="sourceLineNo">708</span> if (length <= 0) {<a name="line.708"></a>
-<span class="sourceLineNo">709</span> LOG.warn("File {} might be still open, length is 0", path);<a name="line.709"></a>
-<span class="sourceLineNo">710</span> }<a name="line.710"></a>
-<span class="sourceLineNo">711</span><a name="line.711"></a>
-<span class="sourceLineNo">712</span> try {<a name="line.712"></a>
-<span class="sourceLineNo">713</span> FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);<a name="line.713"></a>
-<span class="sourceLineNo">714</span> try {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> in = getReader(path, reporter);<a name="line.715"></a>
-<span class="sourceLineNo">716</span> } catch (EOFException e) {<a name="line.716"></a>
-<span class="sourceLineNo">717</span> if (length <= 0) {<a name="line.717"></a>
-<span class="sourceLineNo">718</span> // TODO should we ignore an empty, not-last log file if skip.errors<a name="line.718"></a>
-<span class="sourceLineNo">719</span> // is false? Either way, the caller should decide what to do. E.g.<a name="line.719"></a>
-<span class="sourceLineNo">720</span> // ignore if this is the last log in sequence.<a name="line.720"></a>
-<span class="sourceLineNo">721</span> // TODO is this scenario still possible if the log has been<a name="line.721"></a>
-<span class="sourceLineNo">722</span> // recovered (i.e. closed)<a name="line.722"></a>
-<span class="sourceLineNo">723</span> LOG.warn("Could not open {} for reading. File is empty", path, e);<a name="line.723"></a>
-<span class="sourceLineNo">724</span> }<a name="line.724"></a>
-<span class="sourceLineNo">725</span> // EOFException being ignored<a name="line.725"></a>
-<span class="sourceLineNo">726</span> return null;<a name="line.726"></a>
-<span class="sourceLineNo">727</span> }<a name="line.727"></a>
-<span class="sourceLineNo">728</span> } catch (IOException e) {<a name="line.728"></a>
-<span class="sourceLineNo">729</span> if (e instanceof FileNotFoundException) {<a name="line.729"></a>
-<span class="sourceLineNo">730</span> // A wal file may not exist anymore. Nothing can be recovered so move on<a name="line.730"></a>
-<span class="sourceLineNo">731</span> LOG.warn("File {} does not exist anymore", path, e);<a name="line.731"></a>
-<span class="sourceLineNo">732</span> return null;<a name="line.732"></a>
-<span class="sourceLineNo">733</span> }<a name="line.733"></a>
-<span class="sourceLineNo">734</span> if (!skipErrors || e instanceof InterruptedIOException) {<a name="line.734"></a>
-<span class="sourceLineNo">735</span> throw e; // Don't mark the file corrupted if interrupted, or not skipErrors<a name="line.735"></a>
-<span class="sourceLineNo">736</span> }<a name="line.736"></a>
-<span class="sourceLineNo">737</span> CorruptedLogFileException t =<a name="line.737"></a>
-<span class="sourceLineNo">738</span> new CorruptedLogFileException("skipErrors=true Could not open wal " +<a name="line.738"></a>
-<span class="sourceLineNo">739</span> path + " ignoring");<a name="line.739"></a>
-<span class="sourceLineNo">740</span> t.initCause(e);<a name="line.740"></a>
-<span class="sourceLineNo">741</span> throw t;<a name="line.741"></a>
-<span class="sourceLineNo">742</span> }<a name="line.742"></a>
-<span class="sourceLineNo">743</span> return in;<a name="line.743"></a>
-<span class="sourceLineNo">744</span> }<a name="line.744"></a>
-<span class="sourceLineNo">745</span><a name="line.745"></a>
-<span class="sourceLineNo">746</span> static private Entry getNextLogLine(Reader in, Path path, boolean skipErrors)<a name="line.746"></a>
-<span class="sourceLineNo">747</span> throws CorruptedLogFileException, IOException {<a name="line.747"></a>
-<span class="sourceLineNo">748</span> try {<a name="line.748"></a>
-<span class="sourceLineNo">749</span> return in.next();<a name="line.749"></a>
-<span class="sourceLineNo">750</span> } catch (EOFException eof) {<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // truncated files are expected if a RS crashes (see HBASE-2643)<a name="line.751"></a>
-<span class="sourceLineNo">752</span> LOG.info("EOF from wal {}. Continuing.", path);<a name="line.752"></a>
-<span class="sourceLineNo">753</span> return null;<a name="line.753"></a>
-<span class="sourceLineNo">754</span> } catch (IOException e) {<a name="line.754"></a>
-<span class="sourceLineNo">755</span> // If the IOE resulted from bad file format,<a name="line.755"></a>
-<span class="sourceLineNo">756</span> // then this problem is idempotent and retrying won't help<a name="line.756"></a>
-<span class="sourceLineNo">757</span> if (e.getCause() != null &&<a name="line.757"></a>
-<span class="sourceLineNo">758</span> (e.getCause() instanceof ParseException ||<a name="line.758"></a>
-<span class="sourceLineNo">759</span> e.getCause() instanceof org.apache.hadoop.fs.ChecksumException)) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> LOG.warn("Parse exception from wal {}. Continuing", path, e);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> return null;<a name="line.761"></a>
-<span class="sourceLineNo">762</span> }<a name="line.762"></a>
-<span class="sourceLineNo">763</span> if (!skipErrors) {<a name="line.763"></a>
-<span class="sourceLineNo">764</span> throw e;<a name="line.764"></a>
-<span class="sourceLineNo">765</span> }<a name="line.765"></a>
-<span class="sourceLineNo">766</span> CorruptedLogFileException t =<a name="line.766"></a>
-<span class="sourceLineNo">767</span> new CorruptedLogFileException("skipErrors=true Ignoring exception" +<a name="line.767"></a>
-<span class="sourceLineNo">768</span> " while parsing wal " + path + ". Marking as corrupted");<a name="line.768"></a>
-<span class="sourceLineNo">769</span> t.initCause(e);<a name="line.769"></a>
-<span class="sourceLineNo">770</span> throw t;<a name="line.770"></a>
-<span class="sourceLineNo">771</span> }<a name="line.771"></a>
-<span class="sourceLineNo">772</span> }<a name="line.772"></a>
-<span class="sourceLineNo">773</span><a name="line.773"></a>
-<span class="sourceLineNo">774</span> /**<a name="line.774"></a>
-<span class="sourceLineNo">775</span> * Create a new {@link Writer} for writing log splits.<a name="line.775"></a>
-<span class="sourceLineNo">776</span> * @return a new Writer instance, caller should close<a name="line.776"></a>
-<span class="sourceLineNo">777</span> */<a name="line.777"></a>
-<span class="sourceLineNo">778</span> protected Writer createWriter(Path logfile)<a name="line.778"></a>
-<span class="sourceLineNo">779</span> throws IOException {<a name="line.779"></a>
-<span class="sourceLineNo">780</span> return walFactory.createRecoveredEditsWriter(fs, logfile);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> }<a name="line.781"></a>
-<span class="sourceLineNo">782</span><a name="line.782"></a>
-<span class="sourceLineNo">783</span> /**<a name="line.783"></a>
-<span class="sourceLineNo">784</span> * Create a new {@link Reader} for reading logs to split.<a name="line.784"></a>
-<span class="sourceLineNo">785</span> * @return new Reader instance, caller should close<a name="line.785"></a>
-<span class="sourceLineNo">786</span> */<a name="line.786"></a>
-<span class="sourceLineNo">787</span> protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {<a name="line.787"></a>
-<span class="sourceLineNo">788</span> return walFactory.createReader(fs, curLogFile, reporter);<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> /**<a name="line.791"></a>
-<span class="sourceLineNo">792</span> * Get current open writers<a name="line.792"></a>
-<span class="sourceLineNo">793</span> */<a name="line.793"></a>
-<span class="sourceLineNo">794</span> private int getNumOpenWriters() {<a name="line.794"></a>
-<span class="sourceLineNo">795</span> int result = 0;<a name="line.795"></a>
-<span class="sourceLineNo">796</span> if (this.outputSink != null) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> result += this.outputSink.getNumOpenWriters();<a name="line.797"></a>
-<span class="sourceLineNo">798</span> }<a name="line.798"></a>
-<span class="sourceLineNo">799</span> return result;<a name="line.799"></a>
-<span class="sourceLineNo">800</span> }<a name="line.800"></a>
-<span class="sourceLineNo">801</span><a name="line.801"></a>
-<span class="sourceLineNo">802</span> /**<a name="line.802"></a>
-<span class="sourceLineNo">803</span> * Contains some methods to control WAL-entries producer / consumer interactions<a name="line.803"></a>
-<span class="sourceLineNo">804</span> */<a name="line.804"></a>
-<span class="sourceLineNo">805</span> public static class PipelineController {<a name="line.805"></a>
-<span class="sourceLineNo">806</span> // If an exception is thrown by one of the other threads, it will be<a name="line.806"></a>
-<span class="sourceLineNo">807</span> // stored here.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> AtomicReference<Throwable> thrown = new AtomicReference<>();<a name="line.808"></a>
-<span class="sourceLineNo">809</span><a name="line.809"></a>
-<span class="sourceLineNo">810</span> // Wait/notify for when data has been produced by the writer thread,<a name="line.810"></a>
-<span class="sourceLineNo">811</span> // consumed by the reader thread, or an exception occurred<a name="line.811"></a>
-<span class="sourceLineNo">812</span> public final Object dataAvailable = new Object();<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> void writerThreadError(Throwable t) {<a name="line.814"></a>
-<span class="sourceLineNo">815</span> thrown.compareAndSet(null, t);<a name="line.815"></a>
-<span class="sourceLineNo">816</span> }<a name="line.816"></a>
-<span class="sourceLineNo">817</span><a name="line.817"></a>
-<span class="sourceLineNo">818</span> /**<a name="line.818"></a>
-<span class="sourceLineNo">819</span> * Check for errors in the writer threads. If any is found, rethrow it.<a name="line.819"></a>
-<span class="sourceLineNo">820</span> */<a name="line.820"></a>
-<span class="sourceLineNo">821</span> void checkForErrors() throws IOException {<a name="line.821"></a>
-<span class="sourceLineNo">822</span> Throwable thrown = this.thrown.get();<a name="line.822"></a>
-<span class="sourceLineNo">823</span> if (thrown == null) return;<a name="line.823"></a>
-<span class="sourceLineNo">824</span> if (thrown instanceof IOException) {<a name="line.824"></a>
-<span class="sourceLineNo">825</span> throw new IOException(thrown);<a name="line.825"></a>
-<span class="sourceLineNo">826</span> } else {<a name="line.826"></a>
-<span class="sourceLineNo">827</span> throw new RuntimeException(thrown);<a name="line.827"></a>
-<span class="sourceLineNo">828</span> }<a name="line.828"></a>
-<span class="sourceLineNo">829</span> }<a name="line.829"></a>
-<span class="sourceLineNo">830</span> }<a name="line.830"></a>
-<span class="sourceLineNo">831</span><a name="line.831"></a>
-<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
-<span class="sourceLineNo">833</span> * Class which accumulates edits and separates them into a buffer per region<a name="line.833"></a>
-<span class="sourceLineNo">834</span> * while simultaneously accounting RAM usage. Blocks if the RAM usage crosses<a name="line.834"></a>
-<span class="sourceLineNo">835</span> * a predefined threshold.<a name="line.835"></a>
-<span class="sourceLineNo">836</span> *<a name="line.836"></a>
-<span class="sourceLineNo">837</span> * Writer threads then pull region-specific buffers from this class.<a name="line.837"></a>
-<span class="sourceLineNo">838</span> */<a name="line.838"></a>
-<span class="sourceLineNo">839</span> public static class EntryBuffers {<a name="line.839"></a>
-<span class="sourceLineNo">840</span> PipelineController controller;<a name="line.840"></a>
-<span class="sourceLineNo">841</span><a name="line.841"></a>
-<span class="sourceLineNo">842</span> Map<byte[], RegionEntryBuffer> buffers = new TreeMap<>(Bytes.BYTES_COMPARATOR);<a name="line.842"></a>
-<span class="sourceLineNo">843</span><a name="line.843"></a>
-<span class="sourceLineNo">844</span> /* Track which regions are currently in the middle of writing. We don't allow<a name="line.844"></a>
-<span class="sourceLineNo">845</span> an IO thread to pick up bytes from a region if we're already writing<a name="line.845"></a>
-<span class="sourceLineNo">846</span> data for that region in a different IO thread. */<a name="line.846"></a>
-<span class="sourceLineNo">847</span> Set<byte[]> currentlyWriting = new TreeSet<>(Bytes.BYTES_COMPARATOR);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> long totalBuffered = 0;<a name="line.849"></a>
-<span class="sourceLineNo">850</span> long maxHeapUsage;<a name="line.850"></a>
-<span class="sourceLineNo">851</span> boolean splitWriterCreationBounded;<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> public EntryBuffers(PipelineController controller, long maxHeapUsage) {<a name="line.853"></a>
-<span class="sourceLineNo">854</span> this(controller, maxHeapUsage, false);<a name="line.854"></a>
-<span class="sourceLineNo">855</span> }<a name="line.855"></a>
-<span class="sourceLineNo">856</span><a name="line.856"></a>
-<span class="sourceLineNo">857</span> public EntryBuffers(PipelineController controller, long maxHeapUsage,<a name="line.857"></a>
-<span class="sourceLineNo">858</span> boolean splitWriterCreationBounded){<a name="line.858"></a>
-<span class="sourceLineNo">859</span> this.controller = controller;<a name="line.859"></a>
-<span class="sourceLineNo">860</span> this.maxHeapUsage = maxHeapUsage;<a name="line.860"></a>
-<span class="sourceLineNo">861</span> this.splitWriterCreationBounded = splitWriterCreationBounded;<a name="line.861"></a>
-<span class="sourceLineNo">862</span> }<a name="line.862"></a>
-<span class="sourceLineNo">863</span><a name="line.863"></a>
-<span class="sourceLineNo">864</span> /**<a name="line.864"></a>
-<span class="sourceLineNo">865</span> * Append a log entry into the corresponding region buffer.<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Blocks if the total heap usage has crossed the specified threshold.<a name="line.866"></a>
-<span class="sourceLineNo">867</span> *<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * @throws InterruptedException<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * @throws IOException<a name="line.869"></a>
-<span class="sourceLineNo">870</span> */<a name="line.870"></a>
-<span class="sourceLineNo">871</span> public void appendEntry(Entry entry) throws InterruptedException, IOException {<a name="line.871"></a>
-<span class="sourceLineNo">872</span> WALKey key = entry.getKey();<a name="line.872"></a>
-<span class="sourceLineNo">873</span><a name="line.873"></a>
-<span class="sourceLineNo">874</span> RegionEntryBuffer buffer;<a name="line.874"></a>
-<span class="sourceLineNo">875</span> long incrHeap;<a name="line.875"></a>
-<span class="sourceLineNo">876</span> synchronized (this) {<a name="line.876"></a>
-<span class="sourceLineNo">877</span> buffer = buffers.get(key.getEncodedRegionName());<a name="line.877"></a>
-<span class="sourceLineNo">878</span> if (buffer == null) {<a name="line.878"></a>
-<span class="sourceLineNo">879</span> buffer = new RegionEntryBuffer(key.getTableName(), key.getEncodedRegionName());<a name="line.879"></a>
-<span class="sourceLineNo">880</span> buffers.put(key.getEncodedRegionName(), buffer);<a name="line.880"></a>
-<span class="sourceLineNo">881</span> }<a name="line.881"></a>
-<span class="sourceLineNo">882</span> incrHeap= buffer.appendEntry(entry);<a name="line.882"></a>
-<span class="sourceLineNo">883</span> }<a name="line.883"></a>
-<span class="sourceLineNo">884</span><a name="line.884"></a>
-<span class="sourceLineNo">885</span> // If we crossed the chunk threshold, wait for more space to be available<a name="line.885"></a>
-<span class="sourceLineNo">886</span> synchronized (controller.dataAvailable) {<a name="line.886"></a>
-<span class="sourceLineNo">887</span> totalBuffered += incrHeap;<a name="line.887"></a>
-<span class="sourceLineNo">888</span> while (totalBuffered > maxHeapUsage && controller.thrown.get() == null) {<a name="line.888"></a>
-<span class="sourceLineNo">889</span> LOG.debug("Used {} bytes of buffered edits, waiting for IO threads", totalBuffered);<a name="line.889"></a>
-<span class="sourceLineNo">890</span> controller.dataAvailable.wait(2000);<a name="line.890"></a>
-<span class="sourceLineNo">891</span> }<a name="line.891"></a>
-<span class="sourceLineNo">892</span> controller.dataAvailable.notifyAll();<a name="line.892"></a>
-<span class="sourceLineNo">893</span> }<a name="line.893"></a>
-<span class="sourceLineNo">894</span> controller.checkForErrors();<a name="line.894"></a>
-<span class="sourceLineNo">895</span> }<a name="line.895"></a>
-<span class="sourceLineNo">896</span><a name="line.896"></a>
-<span class="sourceLineNo">897</span> /**<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @return RegionEntryBuffer a buffer of edits to be written.<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> synchronized RegionEntryBuffer getChunkToWrite() {<a name="line.900"></a>
-<span class="sourceLineNo">901</span> // The core part of limiting opening writers is it doesn't return chunk only if the<a name="line.901"></a>
-<span class="sourceLineNo">902</span> // heap size is over maxHeapUsage. Thus it doesn't need to create a writer for each<a name="line.902"></a>
-<span class="sourceLineNo">903</span> // region during splitting. It will flush all the logs in the buffer after splitting<a name="line.903"></a>
-<span class="sourceLineNo">904</span> // through a threadpool, which means the number of writers it created is under control.<a name="line.904"></a>
-<span class="sourceLineNo">905</span> if (splitWriterCreationBounded && totalBuffered < maxHeapUsage) {<a name="line.905"></a>
-<span class="sourceLineNo">906</span> return null;<a name="line.906"></a>
-<span class="sourceLineNo">907</span> }<a name="line.907"></a>
-<span class="sourceLineNo">908</span> long biggestSize = 0;<a name="line.908"></a>
-<span class="sourceLineNo">909</span> byte[] biggestBufferKey = null;<a name="line.909"></a>
-<span class="sourceLineNo">910</span><a name="line.910"></a>
-<span class="sourceLineNo">911</span> for (Map.Entry<byte[], RegionEntryBuffer> entry : buffers.entrySet()) {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> long size = entry.getValue().heapSize();<a name="line.912"></a>
-<span class="sourceLineNo">913</span> if (size > biggestSize && (!currentlyWriting.contains(entry.getKey()))) {<a name="line.913"></a>
-<span class="sourceLineNo">914</span> biggestSize = size;<a name="line.914"></a>
-<span class="sourceLineNo">915</span> biggestBufferKey = entry.getKey();<a name="line.915"></a>
-<span class="sourceLineNo">916</span> }<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span> if (biggestBufferKey == null) {<a name="line.918"></a>
-<span class="sourceLineNo">919</span> return null;<a name="line.919"></a>
-<span class="sourceLineNo">920</span> }<a name="line.920"></a>
-<span class="sourceLineNo">921</span><a name="line.921"></a>
-<span class="sourceLineNo">922</span> RegionEntryBuffer buffer = buffers.remove(biggestBufferKey);<a name="line.922"></a>
-<span class="sourceLineNo">923</span> currentlyWriting.add(biggestBufferKey);<a name="line.923"></a>
-<span class="sourceLineNo">924</span> return buffer;<a name="line.924"></a>
-<span class="sourceLineNo">925</span> }<a name="line.925"></a>
-<span class="sourceLineNo">926</span><a name="line.926"></a>
-<span class="sourceLineNo">927</span> void doneWriting(RegionEntryBuffer buffer) {<a name="line.927"></a>
-<span class="sourceLineNo">928</span> synchronized (this) {<a name="line.928"></a>
-<span class="sourceLineNo">929</span> boolean removed = currentlyWriting.remove(buffer.encodedRegionName);<a name="line.929"></a>
-<span class="sourceLineNo">930</span> assert removed;<a name="line.930"></a>
-<span class="sourceLineNo">931</span> }<a name="line.931"></a>
-<span class="sourceLineNo">932</span> long size = buffer.heapSize();<a name="line.932"></a>
-<span class="sourceLineNo">933</span><a name="line.933"></a>
-<span class="sourceLineNo">934</span> synchronized (controller.dataAvailable) {<a name="line.934"></a>
-<span class="sourceLineNo">935</span> totalBuffered -= size;<a name="line.935"></a>
-<span class="sourceLineNo">936</span> // We may unblock writers<a name="line.936"></a>
-<span class="sourceLineNo">937</span> controller.dataAvailable.notifyAll();<a name="line.937"></a>
-<span class="sourceLineNo">938</span> }<a name="line.938"></a>
-<span class="sourceLineNo">939</span> }<a name="line.939"></a>
-<span class="sourceLineNo">940</span><a name="line.940"></a>
-<span class="sourceLineNo">941</span> synchronized boolean isRegionCurrentlyWriting(byte[] region) {<a name="line.941"></a>
-<span class="sourceLineNo">942</span> return currentlyWriting.contains(region);<a name="line.942"></a>
-<span class="sourceLineNo">943</span> }<a name="line.943"></a>
-<span class="sourceLineNo">944</span><a name="line.944"></a>
-<span class="sourceLineNo">945</span> public void waitUntilDrained() {<a name="line.945"></a>
-<span class="sourceLineNo">946</span> synchronized (controller.dataAvailable) {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> while (totalBuffered > 0) {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> try {<a name="line.948"></a>
-<span class="sourceLineNo">949</span> controller.dataAvailable.wait(2000);<a name="line.949"></a>
-<span class="sourceLineNo">950</span> } catch (InterruptedException e) {<a name="line.950"></a>
-<span class="sourceLineNo">951</span> LOG.warn("Got interrupted while waiting for EntryBuffers is drained");<a name="line.951"></a>
-<span class="sourceLineNo">952</span> Thread.interrupted();<a name="line.952"></a>
-<span class="sourceLineNo">953</span> break;<a name="line.953"></a>
-<span class="sourceLineNo">954</span> }<a name="line.954"></a>
-<span class="sourceLineNo">955</span> }<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span> }<a name="line.958"></a>
-<span class="sourceLineNo">959</span><a name="line.959"></a>
-<span class="sourceLineNo">960</span> /**<a name="line.960"></a>
-<span class="sourceLineNo">961</span> * A buffer of some number of edits for a given region.<a name="line.961"></a>
-<span class="sourceLineNo">962</span> * This accumulates edits and also provides a memory optimization in order to<a name="line.962"></a>
-<span class="sourceLineNo">963</span> * share a single byte array instance for the table and region name.<a name="line.963"></a>
-<span class="sourceLineNo">964</span> * Also tracks memory usage of the accumulated edits.<a name="line.964"></a>
-<span class="sourceLineNo">965</span> */<a name="line.965"></a>
-<span class="sourceLineNo">966</span> public static class RegionEntryBuffer implements HeapSize {<a name="line.966"></a>
-<span class="sourceLineNo">967</span> long heapInBuffer = 0;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> List<Entry> entryBuffer;<a name="line.968"></a>
-<span class="sourceLineNo">969</span> TableName tableName;<a name="line.969"></a>
-<span class="sourceLineNo">970</span> byte[] encodedRegionName;<a name="line.970"></a>
-<span class="sourceLineNo">971</span><a name="line.971"></a>
-<span class="sourceLineNo">972</span> RegionEntryBuffer(TableName tableName, byte[] region) {<a name="line.972"></a>
-<span class="sourceLineNo">973</span> this.tableName = tableName;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> this.encodedRegionName = region;<a name="line.974"></a>
-<span class="sourceLineNo">975</span> this.entryBuffer = new ArrayList<>();<a name="line.975"></a>
-<span class="sourceLineNo">976</span> }<a name="line.976"></a>
-<span class="sourceLineNo">977</span><a name="line.977"></a>
-<span class="sourceLineNo">978</span> long appendEntry(Entry entry) {<a name="line.978"></a>
-<span class="sourceLineNo">979</span> internify(entry);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> entryBuffer.add(entry);<a name="line.980"></a>
-<span class="sourceLineNo">981</span> long incrHeap = entry.getEdit().heapSize() +<a name="line.981"></a>
-<span class="sourceLineNo">982</span> ClassSize.align(2 * ClassSize.REFERENCE) + // WALKey pointers<a name="line.982"></a>
-<span class="sourceLineNo">983</span> 0; // TODO linkedlist entry<a name="line.983"></a>
-<span class="sourceLineNo">984</span> heapInBuffer += incrHeap;<a name="line.984"></a>
-<span class="sourceLineNo">985</span> return incrHeap;<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> private void internify(Entry entry) {<a name="line.988"></a>
-<span class="sourceLineNo">989</span> WALKeyImpl k = entry.getKey();<a name="line.989"></a>
-<span class="sourceLineNo">990</span> k.internTableName(this.tableName);<a name="line.990"></a>
-<span class="sourceLineNo">991</span> k.internEncodedRegionName(this.encodedRegionName);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> }<a name="line.992"></a>
-<span class="sourceLineNo">993</span><a name="line.993"></a>
-<span class="sourceLineNo">994</span> @Override<a name="line.994"></a>
-<span class="sourceLineNo">995</span> public long heapSize() {<a name="line.995"></a>
-<span class="sourceLineNo">996</span> return heapInBuffer;<a name="line.996"></a>
-<span class="sourceLineNo">997</span> }<a name="line.997"></a>
-<span class="sourceLineNo">998</span><a name="line.998"></a>
-<span class="sourceLineNo">999</span> public byte[] getEncodedRegionName() {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> return encodedRegionName;<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> }<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span><a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> public List<Entry> getEntryBuffer() {<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> return entryBuffer;<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> }<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span><a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> public TableName getTableName() {<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return tableName;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span> }<a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> }<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span><a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> public static class WriterThread extends Thread {<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> private volatile boolean shouldStop = false;<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> private PipelineController controller;<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> private EntryBuffers entryBuffers;<a name="line.1015"></a>
-<span class="sourceLineNo">1016</span> private OutputSink outputSink = null;<a name="line.1016"></a>
-<span class="sourceLineNo">1017</span><a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> WriterThread(PipelineController controller, EntryBuffers entryBuffers, OutputSink sink, int i){<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> super(Thread.currentThread().getName() + "-Writer-" + i);<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> this.controller = controller;<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> this.entryBuffers = entryBuffers;<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> outputSink = sink;<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> }<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span><a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> @Override<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> public void run() {<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span> try {<a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> doRun();<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span> } catch (Throwable t) {<a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> LOG.error("Exiting thread", t);<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> controller.writerThreadError(t);<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> }<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> }<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span><a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> private void doRun() throws IOException {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> LOG.trace("Writer thread starting");<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> while (true) {<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span> RegionEntryBuffer buffer = entryBuffers.getChunkToWrite();<a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> if (buffer == null) {<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> // No data currently available, wait on some more to show up<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> synchronized (controller.dataAvailable) {<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span> if (shouldStop && !this.outputSink.flush()) {<a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> return;<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> try {<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span> controller.dataAvailable.wait(500);<a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> } catch (InterruptedException ie) {<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> if (!shouldStop) {<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> throw new RuntimeException(ie);<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span> }<a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> }<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> }<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> continue;<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> }<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span><a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> assert buffer != null;<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> try {<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> writeBuffer(buffer);<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> } finally {<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> entryBuffers.doneWriting(buffer);<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span> }<a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> }<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> }<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span><a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> private void writeBuffer(RegionEntryBuffer buffer) throws IOException {<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span> outputSink.append(buffer);<a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span><a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> void finish() {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> synchronized (controller.dataAvailable) {<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> shouldStop = true;<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span> controller.dataAvailable.notifyAll();<a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> }<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> }<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> }<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span><a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> /**<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span> * The following class is an abstraction class to provide a common interface to support<a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> * different ways of consuming recovered edits.<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> */<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> public static abstract class OutputSink {<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span><a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> protected PipelineController controller;<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> protected EntryBuffers entryBuffers;<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span><a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> protected ConcurrentHashMap<String, SinkWriter> writers = new ConcurrentHashMap<>();<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span> protected final ConcurrentHashMap<String, Long> regionMaximumEditLogSeqNum =<a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> new ConcurrentHashMap<>();<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span><a name="line.1089"></a>
+<span class="sourceLineNo">460</span> * @param logEntry<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * @param fileNameBeingSplit the file being split currently. Used to generate tmp file name.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> * @param conf<a name="line.462"></a>
+<span class="sourceLineNo">463</span> * @return Path to file into which to dump split log edits.<a name="line.463"></a>
+<span class="sourceLineNo">464</span> * @throws IOException<a name="line.464"></a>
+<span class="sourceLineNo">465</span> */<a name="line.465"></a>
+<span class="sourceLineNo">466</span> @SuppressWarnings("deprecation")<a name="line.466"></a>
+<span class="sourceLineNo">467</span> @VisibleForTesting<a name="line.467"></a>
+<span class="sourceLineNo">468</span> static Path getRegionSplitEditsPath(final Entry logEntry, String fileNameBeingSplit,<a name="line.468"></a>
+<span class="sourceLineNo">469</span> Configuration conf)<a name="line.469"></a>
+<span class="sourceLineNo">470</span> throws IOException {<a name="line.470"></a>
+<span class="sourceLineNo">471</span> FileSystem fs = FileSystem.get(conf);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> Path rootDir = FSUtils.getRootDir(conf);<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTableName());<a name="line.473"></a>
+<span class="sourceLineNo">474</span> String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Path dir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span><a name="line.477"></a>
+<span class="sourceLineNo">478</span> if (!fs.exists(regiondir)) {<a name="line.478"></a>
+<span class="sourceLineNo">479</span> LOG.info("This region's directory does not exist: {}."<a name="line.479"></a>
+<span class="sourceLineNo">480</span> + "It is very likely that it was already split so it is "<a name="line.480"></a>
+<span class="sourceLineNo">481</span> + "safe to discard those edits.", regiondir);<a name="line.481"></a>
+<span class="sourceLineNo">482</span> return null;<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span> if (fs.exists(dir) && fs.isFile(dir)) {<a name="line.484"></a>
+<span class="sourceLineNo">485</span> Path tmp = new Path("/tmp");<a name="line.485"></a>
+<span class="sourceLineNo">486</span> if (!fs.exists(tmp)) {<a name="line.486"></a>
+<span class="sourceLineNo">487</span> fs.mkdirs(tmp);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> tmp = new Path(tmp,<a name="line.489"></a>
+<span class="sourceLineNo">490</span> HConstants.RECOVERED_EDITS_DIR + "_" + encodedRegionName);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> LOG.warn("Found existing old file: {}. It could be some "<a name="line.491"></a>
+<span class="sourceLineNo">492</span> + "leftover of an old installation. It should be a folder instead. "<a name="line.492"></a>
+<span class="sourceLineNo">493</span> + "So moving it to {}", dir, tmp);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> if (!fs.rename(dir, tmp)) {<a name="line.494"></a>
+<span class="sourceLineNo">495</span> LOG.warn("Failed to sideline old file {}", dir);<a name="line.495"></a>
+<span class="sourceLineNo">496</span> }<a name="line.496"></a>
+<span class="sourceLineNo">497</span> }<a name="line.497"></a>
+<span class="sourceLineNo">498</span><a name="line.498"></a>
+<span class="sourceLineNo">499</span> if (!fs.exists(dir) && !fs.mkdirs(dir)) {<a name="line.499"></a>
+<span class="sourceLineNo">500</span> LOG.warn("mkdir failed on {}", dir);<a name="line.500"></a>
+<span class="sourceLineNo">501</span> }<a name="line.501"></a>
+<span class="sourceLineNo">502</span> // Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure<a name="line.503"></a>
+<span class="sourceLineNo">504</span> // region's replayRecoveredEdits will not delete it<a name="line.504"></a>
+<span class="sourceLineNo">505</span> String fileName = formatRecoveredEditsFileName(logEntry.getKey().getSequenceId());<a name="line.505"></a>
+<span class="sourceLineNo">506</span> fileName = getTmpRecoveredEditsFileName(fileName + "-" + fileNameBeingSplit);<a name="line.506"></a>
+<span class="sourceLineNo">507</span> return new Path(dir, fileName);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> }<a name="line.508"></a>
+<span class="sourceLineNo">509</span><a name="line.509"></a>
+<span class="sourceLineNo">510</span> private static String getTmpRecoveredEditsFileName(String fileName) {<a name="line.510"></a>
+<span class="sourceLineNo">511</span> return fileName + RECOVERED_LOG_TMPFILE_SUFFIX;<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> /**<a name="line.514"></a>
+<span class="sourceLineNo">515</span> * Get the completed recovered edits file path, renaming it to be by last edit<a name="line.515"></a>
+<span class="sourceLineNo">516</span> * in the file from its first edit. Then we could use the name to skip<a name="line.516"></a>
+<span class="sourceLineNo">517</span> * recovered edits when doing {@link HRegion#replayRecoveredEditsIfAny}.<a name="line.517"></a>
+<span class="sourceLineNo">518</span> * @param srcPath<a name="line.518"></a>
+<span class="sourceLineNo">519</span> * @param maximumEditLogSeqNum<a name="line.519"></a>
+<span class="sourceLineNo">520</span> * @return dstPath take file's last edit log seq num as the name<a name="line.520"></a>
+<span class="sourceLineNo">521</span> */<a name="line.521"></a>
+<span class="sourceLineNo">522</span> private static Path getCompletedRecoveredEditsFilePath(Path srcPath,<a name="line.522"></a>
+<span class="sourceLineNo">523</span> long maximumEditLogSeqNum) {<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String fileName = formatRecoveredEditsFileName(maximumEditLogSeqNum);<a name="line.524"></a>
+<span class="sourceLineNo">525</span> return new Path(srcPath.getParent(), fileName);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> }<a name="line.526"></a>
+<span class="sourceLineNo">527</span><a name="line.527"></a>
+<span class="sourceLineNo">528</span> @VisibleForTesting<a name="line.528"></a>
+<span class="sourceLineNo">529</span> static String formatRecoveredEditsFileName(final long seqid) {<a name="line.529"></a>
+<span class="sourceLineNo">530</span> return String.format("%019d", seqid);<a name="line.530"></a>
+<span class="sourceLineNo">531</span> }<a name="line.531"></a>
+<span class="sourceLineNo">532</span><a name="line.532"></a>
+<span class="sourceLineNo">533</span> private static final Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");<a name="line.533"></a>
+<span class="sourceLineNo">534</span> private static final String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";<a name="line.534"></a>
+<span class="sourceLineNo">535</span><a name="line.535"></a>
+<span class="sourceLineNo">536</span> /**<a name="line.536"></a>
+<span class="sourceLineNo">537</span> * @param regiondir<a name="line.537"></a>
+<span class="sourceLineNo">538</span> * This regions directory in the filesystem.<a name="line.538"></a>
+<span class="sourceLineNo">539</span> * @return The directory that holds recovered edits files for the region<a name="line.539"></a>
+<span class="sourceLineNo">540</span> * <code>regiondir</code><a name="line.540"></a>
+<span class="sourceLineNo">541</span> */<a name="line.541"></a>
+<span class="sourceLineNo">542</span> public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);<a name="line.543"></a>
+<span class="sourceLineNo">544</span> }<a name="line.544"></a>
+<span class="sourceLineNo">545</span><a name="line.545"></a>
+<span class="sourceLineNo">546</span> /**<a name="line.546"></a>
+<span class="sourceLineNo">547</span> * Returns sorted set of edit files made by splitter, excluding files<a name="line.547"></a>
+<span class="sourceLineNo">548</span> * with '.temp' suffix.<a name="line.548"></a>
+<span class="sourceLineNo">549</span> *<a name="line.549"></a>
+<span class="sourceLineNo">550</span> * @param fs<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * @param regiondir<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @return Files in passed <code>regiondir</code> as a sorted set.<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @throws IOException<a name="line.553"></a>
+<span class="sourceLineNo">554</span> */<a name="line.554"></a>
+<span class="sourceLineNo">555</span> public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,<a name="line.555"></a>
+<span class="sourceLineNo">556</span> final Path regiondir) throws IOException {<a name="line.556"></a>
+<span class="sourceLineNo">557</span> NavigableSet<Path> filesSorted = new TreeSet<>();<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Path editsdir = getRegionDirRecoveredEditsDir(regiondir);<a name="line.558"></a>
+<span class="sourceLineNo">559</span> if (!fs.exists(editsdir)) {<a name="line.559"></a>
+<span class="sourceLineNo">560</span> return filesSorted;<a name="line.560"></a>
+<span class="sourceLineNo">561</span> }<a name="line.561"></a>
+<span class="sourceLineNo">562</span> FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> @Override<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public boolean accept(Path p) {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> boolean result = false;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> try {<a name="line.566"></a>
+<span class="sourceLineNo">567</span> // Return files and only files that match the editfile names pattern.<a name="line.567"></a>
+<span class="sourceLineNo">568</span> // There can be other files in this directory other than edit files.<a name="line.568"></a>
+<span class="sourceLineNo">569</span> // In particular, on error, we'll move aside the bad edit file giving<a name="line.569"></a>
+<span class="sourceLineNo">570</span> // it a timestamp suffix. See moveAsideBadEditsFile.<a name="line.570"></a>
+<span class="sourceLineNo">571</span> Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName());<a name="line.571"></a>
+<span class="sourceLineNo">572</span> result = fs.isFile(p) && m.matches();<a name="line.572"></a>
+<span class="sourceLineNo">573</span> // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,<a name="line.573"></a>
+<span class="sourceLineNo">574</span> // because it means splitwal thread is writting this file.<a name="line.574"></a>
+<span class="sourceLineNo">575</span> if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) {<a name="line.575"></a>
+<span class="sourceLineNo">576</span> result = false;<a name="line.576"></a>
+<span class="sourceLineNo">577</span> }<a name="line.577"></a>
+<span class="sourceLineNo">578</span> // Skip SeqId Files<a name="line.578"></a>
+<span class="sourceLineNo">579</span> if (isSequenceIdFile(p)) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> result = false;<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span> } catch (IOException e) {<a name="line.582"></a>
+<span class="sourceLineNo">583</span> LOG.warn("Failed isFile check on {}", p, e);<a name="line.583"></a>
+<span class="sourceLineNo">584</span> }<a name="line.584"></a>
+<span class="sourceLineNo">585</span> return result;<a name="line.585"></a>
+<span class="sourceLineNo">586</span> }<a name="line.586"></a>
+<span class="sourceLineNo">587</span> });<a name="line.587"></a>
+<span class="sourceLineNo">588</span> if (ArrayUtils.isNotEmpty(files)) {<a name="line.588"></a>
+<span class="sourceLineNo">589</span> Arrays.asList(files).forEach(status -> filesSorted.add(status.getPath()));<a name="line.589"></a>
+<span class="sourceLineNo">590</span> }<a name="line.590"></a>
+<span class="sourceLineNo">591</span> return filesSorted;<a name="line.591"></a>
+<span class="sourceLineNo">592</span> }<a name="line.592"></a>
+<span class="sourceLineNo">593</span><a name="line.593"></a>
+<span class="sourceLineNo">594</span> /**<a name="line.594"></a>
+<span class="sourceLineNo">595</span> * Move aside a bad edits file.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> *<a name="line.596"></a>
+<span class="sourceLineNo">597</span> * @param fs<a name="line.597"></a>
+<span class="sourceLineNo">598</span> * @param edits<a name="line.598"></a>
+<span class="sourceLineNo">599</span> * Edits file to move aside.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> * @return The name of the moved aside file.<a name="line.600"></a>
+<span class="sourceLineNo">601</span> * @throws IOException<a name="line.601"></a>
+<span class="sourceLineNo">602</span> */<a name="line.602"></a>
+<span class="sourceLineNo">603</span> public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)<a name="line.603"></a>
+<span class="sourceLineNo">604</span> throws IOException {<a name="line.604"></a>
+<span class="sourceLineNo">605</span> Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."<a name="line.605"></a>
+<span class="sourceLineNo">606</span> + System.currentTimeMillis());<a name="line.606"></a>
+<span class="sourceLineNo">607</span> if (!fs.rename(edits, moveAsideName)) {<a name="line.607"></a>
+<span class="sourceLineNo">608</span> LOG.warn("Rename failed from {} to {}", edits, moveAsideName);<a name="line.608"></a>
+<span class="sourceLineNo">609</span> }<a name="line.609"></a>
+<span class="sourceLineNo">610</span> return moveAsideName;<a name="line.610"></a>
+<span class="sourceLineNo">611</span> }<a name="line.611"></a>
+<span class="sourceLineNo">612</span><a name="line.612"></a>
+<span class="sourceLineNo">613</span> private static final String SEQUENCE_ID_FILE_SUFFIX = ".seqid";<a name="line.613"></a>
+<span class="sourceLineNo">614</span> private static final String OLD_SEQUENCE_ID_FILE_SUFFIX = "_seqid";<a name="line.614"></a>
+<span class="sourceLineNo">615</span> private static final int SEQUENCE_ID_FILE_SUFFIX_LENGTH = SEQUENCE_ID_FILE_SUFFIX.length();<a name="line.615"></a>
+<span class="sourceLineNo">616</span><a name="line.616"></a>
+<span class="sourceLineNo">617</span> /**<a name="line.617"></a>
+<span class="sourceLineNo">618</span> * Is the given file a region open sequence id file.<a name="line.618"></a>
+<span class="sourceLineNo">619</span> */<a name="line.619"></a>
+<span class="sourceLineNo">620</span> @VisibleForTesting<a name="line.620"></a>
+<span class="sourceLineNo">621</span> public static boolean isSequenceIdFile(final Path file) {<a name="line.621"></a>
+<span class="sourceLineNo">622</span> return file.getName().endsWith(SEQUENCE_ID_FILE_SUFFIX)<a name="line.622"></a>
+<span class="sourceLineNo">623</span> || file.getName().endsWith(OLD_SEQUENCE_ID_FILE_SUFFIX);<a name="line.623"></a>
+<span class="sourceLineNo">624</span> }<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> private static FileStatus[] getSequenceIdFiles(FileSystem fs, Path regionDir) throws IOException {<a name="line.626"></a>
+<span class="sourceLineNo">627</span> // TODO: Why are we using a method in here as part of our normal region open where<a name="line.627"></a>
+<span class="sourceLineNo">628</span> // there is no splitting involved? Fix. St.Ack 01/20/2017.<a name="line.628"></a>
+<span class="sourceLineNo">629</span> Path editsDir = WALSplitter.getRegionDirRecoveredEditsDir(regionDir);<a name="line.629"></a>
+<span class="sourceLineNo">630</span> try {<a name="line.630"></a>
+<span class="sourceLineNo">631</span> FileStatus[] files = fs.listStatus(editsDir, WALSplitter::isSequenceIdFile);<a name="line.631"></a>
+<span class="sourceLineNo">632</span> return files != null ? files : new FileStatus[0];<a name="line.632"></a>
+<span class="sourceLineNo">633</span> } catch (FileNotFoundException e) {<a name="line.633"></a>
+<span class="sourceLineNo">634</span> return new FileStatus[0];<a name="line.634"></a>
+<span class="sourceLineNo">635</span> }<a name="line.635"></a>
+<span class="sourceLineNo">636</span> }<a name="line.636"></a>
+<span class="sourceLineNo">637</span><a name="line.637"></a>
+<span class="sourceLineNo">638</span> private static long getMaxSequenceId(FileStatus[] files) {<a name="line.638"></a>
+<span class="sourceLineNo">639</span> long maxSeqId = -1L;<a name="line.639"></a>
+<span class="sourceLineNo">640</span> for (FileStatus file : files) {<a name="line.640"></a>
+<span class="sourceLineNo">641</span> String fileName = file.getPath().getName();<a name="line.641"></a>
+<span class="sourceLineNo">642</span> try {<a name="line.642"></a>
+<span class="sourceLineNo">643</span> maxSeqId = Math.max(maxSeqId, Long<a name="line.643"></a>
+<span class="sourceLineNo">644</span> .parseLong(fileName.substring(0, fileName.length() - SEQUENCE_ID_FILE_SUFFIX_LENGTH)));<a name="line.644"></a>
+<span class="sourceLineNo">645</span> } catch (NumberFormatException ex) {<a name="line.645"></a>
+<span class="sourceLineNo">646</span> LOG.warn("Invalid SeqId File Name={}", fileName);<a name="line.646"></a>
+<span class="sourceLineNo">647</span> }<a name="line.647"></a>
+<span class="sourceLineNo">648</span> }<a name="line.648"></a>
+<span class="sourceLineNo">649</span> return maxSeqId;<a name="line.649"></a>
+<span class="sourceLineNo">650</span> }<a name="line.650"></a>
+<span class="sourceLineNo">651</span><a name="line.651"></a>
+<span class="sourceLineNo">652</span> /**<a name="line.652"></a>
+<span class="sourceLineNo">653</span> * Get the max sequence id which is stored in the region directory. -1 if none.<a name="line.653"></a>
+<span class="sourceLineNo">654</span> */<a name="line.654"></a>
+<span class="sourceLineNo">655</span> public static long getMaxRegionSequenceId(FileSystem fs, Path regionDir) throws IOException {<a name="line.655"></a>
+<span class="sourceLineNo">656</span> return getMaxSequenceId(getSequenceIdFiles(fs, regionDir));<a name="line.656"></a>
+<span class="sourceLineNo">657</span> }<a name="line.657"></a>
+<span class="sourceLineNo">658</span><a name="line.658"></a>
+<span class="sourceLineNo">659</span> /**<a name="line.659"></a>
+<span class="sourceLineNo">660</span> * Create a file with name as region's max sequence id<a name="line.660"></a>
+<span class="sourceLineNo">661</span> */<a name="line.661"></a>
+<span class="sourceLineNo">662</span> public static void writeRegionSequenceIdFile(FileSystem fs, Path regionDir, long newMaxSeqId)<a name="line.662"></a>
+<span class="sourceLineNo">663</span> throws IOException {<a name="line.663"></a>
+<span class="sourceLineNo">664</span> FileStatus[] files = getSequenceIdFiles(fs, regionDir);<a name="line.664"></a>
+<span class="sourceLineNo">665</span> long maxSeqId = getMaxSequenceId(files);<a name="line.665"></a>
+<span class="sourceLineNo">666</span> if (maxSeqId > newMaxSeqId) {<a name="line.666"></a>
+<span class="sourceLineNo">667</span> throw new IOException("The new max sequence id " + newMaxSeqId +<a name="line.667"></a>
+<span class="sourceLineNo">668</span> " is less than the old max sequence id " + maxSeqId);<a name="line.668"></a>
+<span class="sourceLineNo">669</span> }<a name="line.669"></a>
+<span class="sourceLineNo">670</span> // write a new seqId file<a name="line.670"></a>
+<span class="sourceLineNo">671</span> Path newSeqIdFile = new Path(WALSplitter.getRegionDirRecoveredEditsDir(regionDir),<a name="line.671"></a>
+<span class="sourceLineNo">672</span> newMaxSeqId + SEQUENCE_ID_FILE_SUFFIX);<a name="line.672"></a>
+<span class="sourceLineNo">673</span> if (newMaxSeqId != maxSeqId) {<a name="line.673"></a>
+<span class="sourceLineNo">674</span> try {<a name="line.674"></a>
+<span class="sourceLineNo">675</span> if (!fs.createNewFile(newSeqIdFile) && !fs.exists(newSeqIdFile)) {<a name="line.675"></a>
+<span class="sourceLineNo">676</span> throw new IOException("Failed to create SeqId file:" + newSeqIdFile);<a name="line.676"></a>
+<span class="sourceLineNo">677</span> }<a name="line.677"></a>
+<span class="sourceLineNo">678</span> LOG.debug("Wrote file={}, newMaxSeqId={}, maxSeqId={}", newSeqIdFile, newMaxSeqId,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> maxSeqId);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> } catch (FileAlreadyExistsException ignored) {<a name="line.680"></a>
+<span class="sourceLineNo">681</span> // latest hdfs throws this exception. it's all right if newSeqIdFile already exists<a name="line.681"></a>
+<span class="sourceLineNo">682</span> }<
<TRUNCATED>