You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/08/16 14:48:04 UTC
[02/51] [partial] hbase-site git commit: Published site at
092efb42749bf7fc6ad338c96aae8e7b9d3a2c74.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/f3d62514/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
index ee05a1d..06f2ffa 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html
@@ -138,1307 +138,1310 @@
<span class="sourceLineNo">130</span> private Path OLDLOGDIR;<a name="line.130"></a>
<span class="sourceLineNo">131</span> private Path CORRUPTDIR;<a name="line.131"></a>
<span class="sourceLineNo">132</span> private Path TABLEDIR;<a name="line.132"></a>
-<span class="sourceLineNo">133</span><a name="line.133"></a>
-<span class="sourceLineNo">134</span> private static final int NUM_WRITERS = 10;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> private static final int ENTRIES = 10; // entries per writer per region<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> private static final String FILENAME_BEING_SPLIT = "testfile";<a name="line.137"></a>
-<span class="sourceLineNo">138</span> private static final TableName TABLE_NAME =<a name="line.138"></a>
-<span class="sourceLineNo">139</span> TableName.valueOf("t1");<a name="line.139"></a>
-<span class="sourceLineNo">140</span> private static final byte[] FAMILY = Bytes.toBytes("f1");<a name="line.140"></a>
-<span class="sourceLineNo">141</span> private static final byte[] QUALIFIER = Bytes.toBytes("q1");<a name="line.141"></a>
-<span class="sourceLineNo">142</span> private static final byte[] VALUE = Bytes.toBytes("v1");<a name="line.142"></a>
-<span class="sourceLineNo">143</span> private static final String WAL_FILE_PREFIX = "wal.dat.";<a name="line.143"></a>
-<span class="sourceLineNo">144</span> private static List<String> REGIONS = new ArrayList<>();<a name="line.144"></a>
-<span class="sourceLineNo">145</span> private static final String HBASE_SKIP_ERRORS = "hbase.hlog.split.skip.errors";<a name="line.145"></a>
-<span class="sourceLineNo">146</span> private static String ROBBER;<a name="line.146"></a>
-<span class="sourceLineNo">147</span> private static String ZOMBIE;<a name="line.147"></a>
-<span class="sourceLineNo">148</span> private static String [] GROUP = new String [] {"supergroup"};<a name="line.148"></a>
-<span class="sourceLineNo">149</span><a name="line.149"></a>
-<span class="sourceLineNo">150</span> static enum Corruptions {<a name="line.150"></a>
-<span class="sourceLineNo">151</span> INSERT_GARBAGE_ON_FIRST_LINE,<a name="line.151"></a>
-<span class="sourceLineNo">152</span> INSERT_GARBAGE_IN_THE_MIDDLE,<a name="line.152"></a>
-<span class="sourceLineNo">153</span> APPEND_GARBAGE,<a name="line.153"></a>
-<span class="sourceLineNo">154</span> TRUNCATE,<a name="line.154"></a>
-<span class="sourceLineNo">155</span> TRUNCATE_TRAILER<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span><a name="line.157"></a>
-<span class="sourceLineNo">158</span> @BeforeClass<a name="line.158"></a>
-<span class="sourceLineNo">159</span> public static void setUpBeforeClass() throws Exception {<a name="line.159"></a>
-<span class="sourceLineNo">160</span> conf = TEST_UTIL.getConfiguration();<a name="line.160"></a>
-<span class="sourceLineNo">161</span> conf.setClass("hbase.regionserver.hlog.writer.impl",<a name="line.161"></a>
-<span class="sourceLineNo">162</span> InstrumentedLogWriter.class, Writer.class);<a name="line.162"></a>
-<span class="sourceLineNo">163</span> // This is how you turn off shortcircuit read currently. TODO: Fix. Should read config.<a name="line.163"></a>
-<span class="sourceLineNo">164</span> System.setProperty("hbase.tests.use.shortcircuit.reads", "false");<a name="line.164"></a>
-<span class="sourceLineNo">165</span> // Create fake maping user to group and set it to the conf.<a name="line.165"></a>
-<span class="sourceLineNo">166</span> Map<String, String []> u2g_map = new HashMap<>(2);<a name="line.166"></a>
-<span class="sourceLineNo">167</span> ROBBER = User.getCurrent().getName() + "-robber";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> ZOMBIE = User.getCurrent().getName() + "-zombie";<a name="line.168"></a>
-<span class="sourceLineNo">169</span> u2g_map.put(ROBBER, GROUP);<a name="line.169"></a>
-<span class="sourceLineNo">170</span> u2g_map.put(ZOMBIE, GROUP);<a name="line.170"></a>
-<span class="sourceLineNo">171</span> DFSTestUtil.updateConfWithFakeGroupMapping(conf, u2g_map);<a name="line.171"></a>
-<span class="sourceLineNo">172</span> conf.setInt("dfs.heartbeat.interval", 1);<a name="line.172"></a>
-<span class="sourceLineNo">173</span> TEST_UTIL.startMiniDFSCluster(2);<a name="line.173"></a>
-<span class="sourceLineNo">174</span> }<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> @AfterClass<a name="line.176"></a>
-<span class="sourceLineNo">177</span> public static void tearDownAfterClass() throws Exception {<a name="line.177"></a>
-<span class="sourceLineNo">178</span> TEST_UTIL.shutdownMiniDFSCluster();<a name="line.178"></a>
-<span class="sourceLineNo">179</span> }<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> @Rule<a name="line.181"></a>
-<span class="sourceLineNo">182</span> public TestName name = new TestName();<a name="line.182"></a>
-<span class="sourceLineNo">183</span> private WALFactory wals = null;<a name="line.183"></a>
-<span class="sourceLineNo">184</span><a name="line.184"></a>
-<span class="sourceLineNo">185</span> @Before<a name="line.185"></a>
-<span class="sourceLineNo">186</span> public void setUp() throws Exception {<a name="line.186"></a>
-<span class="sourceLineNo">187</span> LOG.info("Cleaning up cluster for new test.");<a name="line.187"></a>
-<span class="sourceLineNo">188</span> fs = TEST_UTIL.getDFSCluster().getFileSystem();<a name="line.188"></a>
-<span class="sourceLineNo">189</span> HBASEDIR = TEST_UTIL.createRootDir();<a name="line.189"></a>
-<span class="sourceLineNo">190</span> HBASELOGDIR = TEST_UTIL.createWALRootDir();<a name="line.190"></a>
-<span class="sourceLineNo">191</span> OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME);<a name="line.192"></a>
-<span class="sourceLineNo">193</span> TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);<a name="line.193"></a>
-<span class="sourceLineNo">194</span> REGIONS.clear();<a name="line.194"></a>
-<span class="sourceLineNo">195</span> Collections.addAll(REGIONS, "bbb", "ccc");<a name="line.195"></a>
-<span class="sourceLineNo">196</span> InstrumentedLogWriter.activateFailure = false;<a name="line.196"></a>
-<span class="sourceLineNo">197</span> wals = new WALFactory(conf, name.getMethodName());<a name="line.197"></a>
-<span class="sourceLineNo">198</span> WALDIR = new Path(HBASELOGDIR,<a name="line.198"></a>
-<span class="sourceLineNo">199</span> AbstractFSWALProvider.getWALDirectoryName(ServerName.valueOf(name.getMethodName(),<a name="line.199"></a>
-<span class="sourceLineNo">200</span> 16010, System.currentTimeMillis()).toString()));<a name="line.200"></a>
-<span class="sourceLineNo">201</span> //fs.mkdirs(WALDIR);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> }<a name="line.202"></a>
-<span class="sourceLineNo">203</span><a name="line.203"></a>
-<span class="sourceLineNo">204</span> @After<a name="line.204"></a>
-<span class="sourceLineNo">205</span> public void tearDown() throws Exception {<a name="line.205"></a>
-<span class="sourceLineNo">206</span> try {<a name="line.206"></a>
-<span class="sourceLineNo">207</span> wals.close();<a name="line.207"></a>
-<span class="sourceLineNo">208</span> } catch(IOException exception) {<a name="line.208"></a>
-<span class="sourceLineNo">209</span> // Some tests will move WALs out from under us. In those cases, we'll get an error on close.<a name="line.209"></a>
-<span class="sourceLineNo">210</span> LOG.info("Ignoring an error while closing down our WALFactory. Fine for some tests, but if" +<a name="line.210"></a>
-<span class="sourceLineNo">211</span> " you see a failure look here.");<a name="line.211"></a>
-<span class="sourceLineNo">212</span> LOG.debug("exception details", exception);<a name="line.212"></a>
-<span class="sourceLineNo">213</span> } finally {<a name="line.213"></a>
-<span class="sourceLineNo">214</span> wals = null;<a name="line.214"></a>
-<span class="sourceLineNo">215</span> fs.delete(HBASEDIR, true);<a name="line.215"></a>
-<span class="sourceLineNo">216</span> fs.delete(HBASELOGDIR, true);<a name="line.216"></a>
-<span class="sourceLineNo">217</span> }<a name="line.217"></a>
-<span class="sourceLineNo">218</span> }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> /**<a name="line.220"></a>
-<span class="sourceLineNo">221</span> * Simulates splitting a WAL out from under a regionserver that is still trying to write it.<a name="line.221"></a>
-<span class="sourceLineNo">222</span> * Ensures we do not lose edits.<a name="line.222"></a>
-<span class="sourceLineNo">223</span> * @throws IOException<a name="line.223"></a>
-<span class="sourceLineNo">224</span> * @throws InterruptedException<a name="line.224"></a>
-<span class="sourceLineNo">225</span> */<a name="line.225"></a>
-<span class="sourceLineNo">226</span> @Test<a name="line.226"></a>
-<span class="sourceLineNo">227</span> public void testLogCannotBeWrittenOnceParsed() throws IOException, InterruptedException {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> final AtomicLong counter = new AtomicLong(0);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> AtomicBoolean stop = new AtomicBoolean(false);<a name="line.229"></a>
-<span class="sourceLineNo">230</span> // Region we'll write edits too and then later examine to make sure they all made it in.<a name="line.230"></a>
-<span class="sourceLineNo">231</span> final String region = REGIONS.get(0);<a name="line.231"></a>
-<span class="sourceLineNo">232</span> final int numWriters = 3;<a name="line.232"></a>
-<span class="sourceLineNo">233</span> Thread zombie = new ZombieLastLogWriterRegionServer(counter, stop, region, numWriters);<a name="line.233"></a>
-<span class="sourceLineNo">234</span> try {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> long startCount = counter.get();<a name="line.235"></a>
-<span class="sourceLineNo">236</span> zombie.start();<a name="line.236"></a>
-<span class="sourceLineNo">237</span> // Wait till writer starts going.<a name="line.237"></a>
-<span class="sourceLineNo">238</span> while (startCount == counter.get()) Threads.sleep(1);<a name="line.238"></a>
-<span class="sourceLineNo">239</span> // Give it a second to write a few appends.<a name="line.239"></a>
-<span class="sourceLineNo">240</span> Threads.sleep(1000);<a name="line.240"></a>
-<span class="sourceLineNo">241</span> final Configuration conf2 = HBaseConfiguration.create(conf);<a name="line.241"></a>
-<span class="sourceLineNo">242</span> final User robber = User.createUserForTesting(conf2, ROBBER, GROUP);<a name="line.242"></a>
-<span class="sourceLineNo">243</span> int count = robber.runAs(new PrivilegedExceptionAction<Integer>() {<a name="line.243"></a>
-<span class="sourceLineNo">244</span> @Override<a name="line.244"></a>
-<span class="sourceLineNo">245</span> public Integer run() throws Exception {<a name="line.245"></a>
-<span class="sourceLineNo">246</span> StringBuilder ls = new StringBuilder("Contents of WALDIR (").append(WALDIR)<a name="line.246"></a>
-<span class="sourceLineNo">247</span> .append("):\n");<a name="line.247"></a>
-<span class="sourceLineNo">248</span> for (FileStatus status : fs.listStatus(WALDIR)) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span> ls.append("\t").append(status.toString()).append("\n");<a name="line.249"></a>
-<span class="sourceLineNo">250</span> }<a name="line.250"></a>
-<span class="sourceLineNo">251</span> LOG.debug(Objects.toString(ls));<a name="line.251"></a>
-<span class="sourceLineNo">252</span> LOG.info("Splitting WALs out from under zombie. Expecting " + numWriters + " files.");<a name="line.252"></a>
-<span class="sourceLineNo">253</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf2, wals);<a name="line.253"></a>
-<span class="sourceLineNo">254</span> LOG.info("Finished splitting out from under zombie.");<a name="line.254"></a>
-<span class="sourceLineNo">255</span> Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);<a name="line.255"></a>
-<span class="sourceLineNo">256</span> assertEquals("wrong number of split files for region", numWriters, logfiles.length);<a name="line.256"></a>
-<span class="sourceLineNo">257</span> int count = 0;<a name="line.257"></a>
-<span class="sourceLineNo">258</span> for (Path logfile: logfiles) {<a name="line.258"></a>
-<span class="sourceLineNo">259</span> count += countWAL(logfile);<a name="line.259"></a>
-<span class="sourceLineNo">260</span> }<a name="line.260"></a>
-<span class="sourceLineNo">261</span> return count;<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span> });<a name="line.263"></a>
-<span class="sourceLineNo">264</span> LOG.info("zombie=" + counter.get() + ", robber=" + count);<a name="line.264"></a>
-<span class="sourceLineNo">265</span> assertTrue("The log file could have at most 1 extra log entry, but can't have less. " +<a name="line.265"></a>
-<span class="sourceLineNo">266</span> "Zombie could write " + counter.get() + " and logfile had only " + count,<a name="line.266"></a>
-<span class="sourceLineNo">267</span> counter.get() == count || counter.get() + 1 == count);<a name="line.267"></a>
-<span class="sourceLineNo">268</span> } finally {<a name="line.268"></a>
-<span class="sourceLineNo">269</span> stop.set(true);<a name="line.269"></a>
-<span class="sourceLineNo">270</span> zombie.interrupt();<a name="line.270"></a>
-<span class="sourceLineNo">271</span> Threads.threadDumpingIsAlive(zombie);<a name="line.271"></a>
-<span class="sourceLineNo">272</span> }<a name="line.272"></a>
-<span class="sourceLineNo">273</span> }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * This thread will keep writing to a 'wal' file even after the split process has started.<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * It simulates a region server that was considered dead but woke up and wrote some more to the<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * last log entry. Does its writing as an alternate user in another filesystem instance to<a name="line.278"></a>
-<span class="sourceLineNo">279</span> * simulate better it being a regionserver.<a name="line.279"></a>
-<span class="sourceLineNo">280</span> */<a name="line.280"></a>
-<span class="sourceLineNo">281</span> class ZombieLastLogWriterRegionServer extends Thread {<a name="line.281"></a>
-<span class="sourceLineNo">282</span> final AtomicLong editsCount;<a name="line.282"></a>
-<span class="sourceLineNo">283</span> final AtomicBoolean stop;<a name="line.283"></a>
-<span class="sourceLineNo">284</span> final int numOfWriters;<a name="line.284"></a>
-<span class="sourceLineNo">285</span> /**<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * Region to write edits for.<a name="line.286"></a>
-<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> final String region;<a name="line.288"></a>
-<span class="sourceLineNo">289</span> final User user;<a name="line.289"></a>
-<span class="sourceLineNo">290</span><a name="line.290"></a>
-<span class="sourceLineNo">291</span> public ZombieLastLogWriterRegionServer(AtomicLong counter, AtomicBoolean stop,<a name="line.291"></a>
-<span class="sourceLineNo">292</span> final String region, final int writers)<a name="line.292"></a>
-<span class="sourceLineNo">293</span> throws IOException, InterruptedException {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> super("ZombieLastLogWriterRegionServer");<a name="line.294"></a>
-<span class="sourceLineNo">295</span> setDaemon(true);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> this.stop = stop;<a name="line.296"></a>
-<span class="sourceLineNo">297</span> this.editsCount = counter;<a name="line.297"></a>
-<span class="sourceLineNo">298</span> this.region = region;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> this.user = User.createUserForTesting(conf, ZOMBIE, GROUP);<a name="line.299"></a>
-<span class="sourceLineNo">300</span> numOfWriters = writers;<a name="line.300"></a>
-<span class="sourceLineNo">301</span> }<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> @Override<a name="line.303"></a>
-<span class="sourceLineNo">304</span> public void run() {<a name="line.304"></a>
-<span class="sourceLineNo">305</span> try {<a name="line.305"></a>
-<span class="sourceLineNo">306</span> doWriting();<a name="line.306"></a>
-<span class="sourceLineNo">307</span> } catch (IOException e) {<a name="line.307"></a>
-<span class="sourceLineNo">308</span> LOG.warn(getName() + " Writer exiting " + e);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> } catch (InterruptedException e) {<a name="line.309"></a>
-<span class="sourceLineNo">310</span> LOG.warn(getName() + " Writer exiting " + e);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> }<a name="line.311"></a>
-<span class="sourceLineNo">312</span> }<a name="line.312"></a>
-<span class="sourceLineNo">313</span><a name="line.313"></a>
-<span class="sourceLineNo">314</span> private void doWriting() throws IOException, InterruptedException {<a name="line.314"></a>
-<span class="sourceLineNo">315</span> this.user.runAs(new PrivilegedExceptionAction<Object>() {<a name="line.315"></a>
-<span class="sourceLineNo">316</span> @Override<a name="line.316"></a>
-<span class="sourceLineNo">317</span> public Object run() throws Exception {<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // Index of the WAL we want to keep open. generateWALs will leave open the WAL whose<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // index we supply here.<a name="line.319"></a>
-<span class="sourceLineNo">320</span> int walToKeepOpen = numOfWriters - 1;<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // The below method writes numOfWriters files each with ENTRIES entries for a total of<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // numOfWriters * ENTRIES added per column family in the region.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> Writer writer = null;<a name="line.323"></a>
-<span class="sourceLineNo">324</span> try {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> writer = generateWALs(numOfWriters, ENTRIES, walToKeepOpen);<a name="line.325"></a>
-<span class="sourceLineNo">326</span> } catch (IOException e1) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> throw new RuntimeException("Failed", e1);<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span> // Update counter so has all edits written so far.<a name="line.329"></a>
-<span class="sourceLineNo">330</span> editsCount.addAndGet(numOfWriters * ENTRIES);<a name="line.330"></a>
-<span class="sourceLineNo">331</span> loop(writer);<a name="line.331"></a>
-<span class="sourceLineNo">332</span> // If we've been interruped, then things should have shifted out from under us.<a name="line.332"></a>
-<span class="sourceLineNo">333</span> // closing should error<a name="line.333"></a>
-<span class="sourceLineNo">334</span> try {<a name="line.334"></a>
-<span class="sourceLineNo">335</span> writer.close();<a name="line.335"></a>
-<span class="sourceLineNo">336</span> fail("Writing closing after parsing should give an error.");<a name="line.336"></a>
-<span class="sourceLineNo">337</span> } catch (IOException exception) {<a name="line.337"></a>
-<span class="sourceLineNo">338</span> LOG.debug("ignoring error when closing final writer.", exception);<a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span> return null;<a name="line.340"></a>
-<span class="sourceLineNo">341</span> }<a name="line.341"></a>
-<span class="sourceLineNo">342</span> });<a name="line.342"></a>
-<span class="sourceLineNo">343</span> }<a name="line.343"></a>
-<span class="sourceLineNo">344</span><a name="line.344"></a>
-<span class="sourceLineNo">345</span> private void loop(final Writer writer) {<a name="line.345"></a>
-<span class="sourceLineNo">346</span> byte [] regionBytes = Bytes.toBytes(this.region);<a name="line.346"></a>
-<span class="sourceLineNo">347</span> while (!stop.get()) {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> try {<a name="line.348"></a>
-<span class="sourceLineNo">349</span> long seq = appendEntry(writer, TABLE_NAME, regionBytes,<a name="line.349"></a>
-<span class="sourceLineNo">350</span> Bytes.toBytes("r" + editsCount.get()), regionBytes, QUALIFIER, VALUE, 0);<a name="line.350"></a>
-<span class="sourceLineNo">351</span> long count = editsCount.incrementAndGet();<a name="line.351"></a>
-<span class="sourceLineNo">352</span> LOG.info(getName() + " sync count=" + count + ", seq=" + seq);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> try {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> Thread.sleep(1);<a name="line.354"></a>
-<span class="sourceLineNo">355</span> } catch (InterruptedException e) {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> //<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span> } catch (IOException ex) {<a name="line.358"></a>
-<span class="sourceLineNo">359</span> LOG.error(getName() + " ex " + ex.toString());<a name="line.359"></a>
-<span class="sourceLineNo">360</span> if (ex instanceof RemoteException) {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> LOG.error("Juliet: got RemoteException " + ex.getMessage() +<a name="line.361"></a>
-<span class="sourceLineNo">362</span> " while writing " + (editsCount.get() + 1));<a name="line.362"></a>
-<span class="sourceLineNo">363</span> } else {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error(getName() + " failed to write....at " + editsCount.get());<a name="line.364"></a>
-<span class="sourceLineNo">365</span> fail("Failed to write " + editsCount.get());<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> break;<a name="line.367"></a>
-<span class="sourceLineNo">368</span> } catch (Throwable t) {<a name="line.368"></a>
-<span class="sourceLineNo">369</span> LOG.error(getName() + " HOW? " + t);<a name="line.369"></a>
-<span class="sourceLineNo">370</span> LOG.debug("exception details", t);<a name="line.370"></a>
-<span class="sourceLineNo">371</span> break;<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
-<span class="sourceLineNo">373</span> }<a name="line.373"></a>
-<span class="sourceLineNo">374</span> LOG.info(getName() + " Writer exiting");<a name="line.374"></a>
-<span class="sourceLineNo">375</span> }<a name="line.375"></a>
-<span class="sourceLineNo">376</span> }<a name="line.376"></a>
-<span class="sourceLineNo">377</span><a name="line.377"></a>
-<span class="sourceLineNo">378</span> /**<a name="line.378"></a>
-<span class="sourceLineNo">379</span> * {@see https://issues.apache.org/jira/browse/HBASE-3020}<a name="line.379"></a>
-<span class="sourceLineNo">380</span> */<a name="line.380"></a>
-<span class="sourceLineNo">381</span> @Test<a name="line.381"></a>
-<span class="sourceLineNo">382</span> public void testRecoveredEditsPathForMeta() throws IOException {<a name="line.382"></a>
-<span class="sourceLineNo">383</span> byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes();<a name="line.383"></a>
-<span class="sourceLineNo">384</span> Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);<a name="line.384"></a>
-<span class="sourceLineNo">385</span> Path regiondir = new Path(tdir,<a name="line.385"></a>
-<span class="sourceLineNo">386</span> RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.386"></a>
-<span class="sourceLineNo">387</span> fs.mkdirs(regiondir);<a name="line.387"></a>
-<span class="sourceLineNo">388</span> long now = System.currentTimeMillis();<a name="line.388"></a>
-<span class="sourceLineNo">389</span> Entry entry =<a name="line.389"></a>
-<span class="sourceLineNo">390</span> new Entry(new WALKeyImpl(encoded,<a name="line.390"></a>
-<span class="sourceLineNo">391</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.391"></a>
-<span class="sourceLineNo">392</span> new WALEdit());<a name="line.392"></a>
-<span class="sourceLineNo">393</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.393"></a>
-<span class="sourceLineNo">394</span> FILENAME_BEING_SPLIT, conf);<a name="line.394"></a>
-<span class="sourceLineNo">395</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.395"></a>
-<span class="sourceLineNo">396</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.396"></a>
-<span class="sourceLineNo">397</span> }<a name="line.397"></a>
-<span class="sourceLineNo">398</span><a name="line.398"></a>
-<span class="sourceLineNo">399</span> /**<a name="line.399"></a>
-<span class="sourceLineNo">400</span> * Test old recovered edits file doesn't break WALSplitter.<a name="line.400"></a>
-<span class="sourceLineNo">401</span> * This is useful in upgrading old instances.<a name="line.401"></a>
-<span class="sourceLineNo">402</span> */<a name="line.402"></a>
-<span class="sourceLineNo">403</span> @Test<a name="line.403"></a>
-<span class="sourceLineNo">404</span> public void testOldRecoveredEditsFileSidelined() throws IOException {<a name="line.404"></a>
-<span class="sourceLineNo">405</span> byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes();<a name="line.405"></a>
-<span class="sourceLineNo">406</span> Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> Path regiondir = new Path(tdir,<a name="line.407"></a>
-<span class="sourceLineNo">408</span> RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.408"></a>
-<span class="sourceLineNo">409</span> fs.mkdirs(regiondir);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> long now = System.currentTimeMillis();<a name="line.410"></a>
-<span class="sourceLineNo">411</span> Entry entry =<a name="line.411"></a>
-<span class="sourceLineNo">412</span> new Entry(new WALKeyImpl(encoded,<a name="line.412"></a>
-<span class="sourceLineNo">413</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.413"></a>
-<span class="sourceLineNo">414</span> new WALEdit());<a name="line.414"></a>
-<span class="sourceLineNo">415</span> Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);<a name="line.415"></a>
-<span class="sourceLineNo">416</span> assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());<a name="line.416"></a>
-<span class="sourceLineNo">417</span> fs.createNewFile(parent); // create a recovered.edits file<a name="line.417"></a>
-<span class="sourceLineNo">418</span><a name="line.418"></a>
-<span class="sourceLineNo">419</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.419"></a>
-<span class="sourceLineNo">420</span> FILENAME_BEING_SPLIT, conf);<a name="line.420"></a>
-<span class="sourceLineNo">421</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.421"></a>
-<span class="sourceLineNo">422</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.422"></a>
-<span class="sourceLineNo">423</span> WALFactory.createRecoveredEditsWriter(fs, p, conf).close();<a name="line.423"></a>
-<span class="sourceLineNo">424</span> }<a name="line.424"></a>
-<span class="sourceLineNo">425</span><a name="line.425"></a>
-<span class="sourceLineNo">426</span> private void useDifferentDFSClient() throws IOException {<a name="line.426"></a>
-<span class="sourceLineNo">427</span> // make fs act as a different client now<a name="line.427"></a>
-<span class="sourceLineNo">428</span> // initialize will create a new DFSClient with a new client ID<a name="line.428"></a>
-<span class="sourceLineNo">429</span> fs.initialize(fs.getUri(), conf);<a name="line.429"></a>
-<span class="sourceLineNo">430</span> }<a name="line.430"></a>
-<span class="sourceLineNo">431</span><a name="line.431"></a>
-<span class="sourceLineNo">432</span> @Test<a name="line.432"></a>
-<span class="sourceLineNo">433</span> public void testSplitPreservesEdits() throws IOException{<a name="line.433"></a>
-<span class="sourceLineNo">434</span> final String REGION = "region__1";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> REGIONS.clear();<a name="line.435"></a>
-<span class="sourceLineNo">436</span> REGIONS.add(REGION);<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> generateWALs(1, 10, -1, 0);<a name="line.438"></a>
-<span class="sourceLineNo">439</span> useDifferentDFSClient();<a name="line.439"></a>
-<span class="sourceLineNo">440</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.440"></a>
-<span class="sourceLineNo">441</span> Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();<a name="line.441"></a>
-<span class="sourceLineNo">442</span> Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);<a name="line.442"></a>
-<span class="sourceLineNo">443</span> assertEquals(1, splitLog.length);<a name="line.443"></a>
-<span class="sourceLineNo">444</span><a name="line.444"></a>
-<span class="sourceLineNo">445</span> assertTrue("edits differ after split", logsAreEqual(originalLog, splitLog[0]));<a name="line.445"></a>
-<span class="sourceLineNo">446</span> }<a name="line.446"></a>
+<span class="sourceLineNo">133</span> private String TMPDIRNAME;<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span> private static final int NUM_WRITERS = 10;<a name="line.135"></a>
+<span class="sourceLineNo">136</span> private static final int ENTRIES = 10; // entries per writer per region<a name="line.136"></a>
+<span class="sourceLineNo">137</span><a name="line.137"></a>
+<span class="sourceLineNo">138</span> private static final String FILENAME_BEING_SPLIT = "testfile";<a name="line.138"></a>
+<span class="sourceLineNo">139</span> private static final TableName TABLE_NAME =<a name="line.139"></a>
+<span class="sourceLineNo">140</span> TableName.valueOf("t1");<a name="line.140"></a>
+<span class="sourceLineNo">141</span> private static final byte[] FAMILY = Bytes.toBytes("f1");<a name="line.141"></a>
+<span class="sourceLineNo">142</span> private static final byte[] QUALIFIER = Bytes.toBytes("q1");<a name="line.142"></a>
+<span class="sourceLineNo">143</span> private static final byte[] VALUE = Bytes.toBytes("v1");<a name="line.143"></a>
+<span class="sourceLineNo">144</span> private static final String WAL_FILE_PREFIX = "wal.dat.";<a name="line.144"></a>
+<span class="sourceLineNo">145</span> private static List<String> REGIONS = new ArrayList<>();<a name="line.145"></a>
+<span class="sourceLineNo">146</span> private static final String HBASE_SKIP_ERRORS = "hbase.hlog.split.skip.errors";<a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static String ROBBER;<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private static String ZOMBIE;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private static String [] GROUP = new String [] {"supergroup"};<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> static enum Corruptions {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> INSERT_GARBAGE_ON_FIRST_LINE,<a name="line.152"></a>
+<span class="sourceLineNo">153</span> INSERT_GARBAGE_IN_THE_MIDDLE,<a name="line.153"></a>
+<span class="sourceLineNo">154</span> APPEND_GARBAGE,<a name="line.154"></a>
+<span class="sourceLineNo">155</span> TRUNCATE,<a name="line.155"></a>
+<span class="sourceLineNo">156</span> TRUNCATE_TRAILER<a name="line.156"></a>
+<span class="sourceLineNo">157</span> }<a name="line.157"></a>
+<span class="sourceLineNo">158</span><a name="line.158"></a>
+<span class="sourceLineNo">159</span> @BeforeClass<a name="line.159"></a>
+<span class="sourceLineNo">160</span> public static void setUpBeforeClass() throws Exception {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> conf = TEST_UTIL.getConfiguration();<a name="line.161"></a>
+<span class="sourceLineNo">162</span> conf.setClass("hbase.regionserver.hlog.writer.impl",<a name="line.162"></a>
+<span class="sourceLineNo">163</span> InstrumentedLogWriter.class, Writer.class);<a name="line.163"></a>
+<span class="sourceLineNo">164</span> // This is how you turn off shortcircuit read currently. TODO: Fix. Should read config.<a name="line.164"></a>
+<span class="sourceLineNo">165</span> System.setProperty("hbase.tests.use.shortcircuit.reads", "false");<a name="line.165"></a>
+<span class="sourceLineNo">166</span> // Create fake maping user to group and set it to the conf.<a name="line.166"></a>
+<span class="sourceLineNo">167</span> Map<String, String []> u2g_map = new HashMap<>(2);<a name="line.167"></a>
+<span class="sourceLineNo">168</span> ROBBER = User.getCurrent().getName() + "-robber";<a name="line.168"></a>
+<span class="sourceLineNo">169</span> ZOMBIE = User.getCurrent().getName() + "-zombie";<a name="line.169"></a>
+<span class="sourceLineNo">170</span> u2g_map.put(ROBBER, GROUP);<a name="line.170"></a>
+<span class="sourceLineNo">171</span> u2g_map.put(ZOMBIE, GROUP);<a name="line.171"></a>
+<span class="sourceLineNo">172</span> DFSTestUtil.updateConfWithFakeGroupMapping(conf, u2g_map);<a name="line.172"></a>
+<span class="sourceLineNo">173</span> conf.setInt("dfs.heartbeat.interval", 1);<a name="line.173"></a>
+<span class="sourceLineNo">174</span> TEST_UTIL.startMiniDFSCluster(2);<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span><a name="line.176"></a>
+<span class="sourceLineNo">177</span> @AfterClass<a name="line.177"></a>
+<span class="sourceLineNo">178</span> public static void tearDownAfterClass() throws Exception {<a name="line.178"></a>
+<span class="sourceLineNo">179</span> TEST_UTIL.shutdownMiniDFSCluster();<a name="line.179"></a>
+<span class="sourceLineNo">180</span> }<a name="line.180"></a>
+<span class="sourceLineNo">181</span><a name="line.181"></a>
+<span class="sourceLineNo">182</span> @Rule<a name="line.182"></a>
+<span class="sourceLineNo">183</span> public TestName name = new TestName();<a name="line.183"></a>
+<span class="sourceLineNo">184</span> private WALFactory wals = null;<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> @Before<a name="line.186"></a>
+<span class="sourceLineNo">187</span> public void setUp() throws Exception {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> LOG.info("Cleaning up cluster for new test.");<a name="line.188"></a>
+<span class="sourceLineNo">189</span> fs = TEST_UTIL.getDFSCluster().getFileSystem();<a name="line.189"></a>
+<span class="sourceLineNo">190</span> HBASEDIR = TEST_UTIL.createRootDir();<a name="line.190"></a>
+<span class="sourceLineNo">191</span> HBASELOGDIR = TEST_UTIL.createWALRootDir();<a name="line.191"></a>
+<span class="sourceLineNo">192</span> OLDLOGDIR = new Path(HBASELOGDIR, HConstants.HREGION_OLDLOGDIR_NAME);<a name="line.192"></a>
+<span class="sourceLineNo">193</span> CORRUPTDIR = new Path(HBASELOGDIR, HConstants.CORRUPT_DIR_NAME);<a name="line.193"></a>
+<span class="sourceLineNo">194</span> TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);<a name="line.194"></a>
+<span class="sourceLineNo">195</span> TMPDIRNAME = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY,<a name="line.195"></a>
+<span class="sourceLineNo">196</span> HConstants.DEFAULT_TEMPORARY_HDFS_DIRECTORY);<a name="line.196"></a>
+<span class="sourceLineNo">197</span> REGIONS.clear();<a name="line.197"></a>
+<span class="sourceLineNo">198</span> Collections.addAll(REGIONS, "bbb", "ccc");<a name="line.198"></a>
+<span class="sourceLineNo">199</span> InstrumentedLogWriter.activateFailure = false;<a name="line.199"></a>
+<span class="sourceLineNo">200</span> wals = new WALFactory(conf, name.getMethodName());<a name="line.200"></a>
+<span class="sourceLineNo">201</span> WALDIR = new Path(HBASELOGDIR,<a name="line.201"></a>
+<span class="sourceLineNo">202</span> AbstractFSWALProvider.getWALDirectoryName(ServerName.valueOf(name.getMethodName(),<a name="line.202"></a>
+<span class="sourceLineNo">203</span> 16010, System.currentTimeMillis()).toString()));<a name="line.203"></a>
+<span class="sourceLineNo">204</span> //fs.mkdirs(WALDIR);<a name="line.204"></a>
+<span class="sourceLineNo">205</span> }<a name="line.205"></a>
+<span class="sourceLineNo">206</span><a name="line.206"></a>
+<span class="sourceLineNo">207</span> @After<a name="line.207"></a>
+<span class="sourceLineNo">208</span> public void tearDown() throws Exception {<a name="line.208"></a>
+<span class="sourceLineNo">209</span> try {<a name="line.209"></a>
+<span class="sourceLineNo">210</span> wals.close();<a name="line.210"></a>
+<span class="sourceLineNo">211</span> } catch(IOException exception) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span> // Some tests will move WALs out from under us. In those cases, we'll get an error on close.<a name="line.212"></a>
+<span class="sourceLineNo">213</span> LOG.info("Ignoring an error while closing down our WALFactory. Fine for some tests, but if" +<a name="line.213"></a>
+<span class="sourceLineNo">214</span> " you see a failure look here.");<a name="line.214"></a>
+<span class="sourceLineNo">215</span> LOG.debug("exception details", exception);<a name="line.215"></a>
+<span class="sourceLineNo">216</span> } finally {<a name="line.216"></a>
+<span class="sourceLineNo">217</span> wals = null;<a name="line.217"></a>
+<span class="sourceLineNo">218</span> fs.delete(HBASEDIR, true);<a name="line.218"></a>
+<span class="sourceLineNo">219</span> fs.delete(HBASELOGDIR, true);<a name="line.219"></a>
+<span class="sourceLineNo">220</span> }<a name="line.220"></a>
+<span class="sourceLineNo">221</span> }<a name="line.221"></a>
+<span class="sourceLineNo">222</span><a name="line.222"></a>
+<span class="sourceLineNo">223</span> /**<a name="line.223"></a>
+<span class="sourceLineNo">224</span> * Simulates splitting a WAL out from under a regionserver that is still trying to write it.<a name="line.224"></a>
+<span class="sourceLineNo">225</span> * Ensures we do not lose edits.<a name="line.225"></a>
+<span class="sourceLineNo">226</span> * @throws IOException<a name="line.226"></a>
+<span class="sourceLineNo">227</span> * @throws InterruptedException<a name="line.227"></a>
+<span class="sourceLineNo">228</span> */<a name="line.228"></a>
+<span class="sourceLineNo">229</span> @Test<a name="line.229"></a>
+<span class="sourceLineNo">230</span> public void testLogCannotBeWrittenOnceParsed() throws IOException, InterruptedException {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> final AtomicLong counter = new AtomicLong(0);<a name="line.231"></a>
+<span class="sourceLineNo">232</span> AtomicBoolean stop = new AtomicBoolean(false);<a name="line.232"></a>
+<span class="sourceLineNo">233</span> // Region we'll write edits too and then later examine to make sure they all made it in.<a name="line.233"></a>
+<span class="sourceLineNo">234</span> final String region = REGIONS.get(0);<a name="line.234"></a>
+<span class="sourceLineNo">235</span> final int numWriters = 3;<a name="line.235"></a>
+<span class="sourceLineNo">236</span> Thread zombie = new ZombieLastLogWriterRegionServer(counter, stop, region, numWriters);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> try {<a name="line.237"></a>
+<span class="sourceLineNo">238</span> long startCount = counter.get();<a name="line.238"></a>
+<span class="sourceLineNo">239</span> zombie.start();<a name="line.239"></a>
+<span class="sourceLineNo">240</span> // Wait till writer starts going.<a name="line.240"></a>
+<span class="sourceLineNo">241</span> while (startCount == counter.get()) Threads.sleep(1);<a name="line.241"></a>
+<span class="sourceLineNo">242</span> // Give it a second to write a few appends.<a name="line.242"></a>
+<span class="sourceLineNo">243</span> Threads.sleep(1000);<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Configuration conf2 = HBaseConfiguration.create(conf);<a name="line.244"></a>
+<span class="sourceLineNo">245</span> final User robber = User.createUserForTesting(conf2, ROBBER, GROUP);<a name="line.245"></a>
+<span class="sourceLineNo">246</span> int count = robber.runAs(new PrivilegedExceptionAction<Integer>() {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> @Override<a name="line.247"></a>
+<span class="sourceLineNo">248</span> public Integer run() throws Exception {<a name="line.248"></a>
+<span class="sourceLineNo">249</span> StringBuilder ls = new StringBuilder("Contents of WALDIR (").append(WALDIR)<a name="line.249"></a>
+<span class="sourceLineNo">250</span> .append("):\n");<a name="line.250"></a>
+<span class="sourceLineNo">251</span> for (FileStatus status : fs.listStatus(WALDIR)) {<a name="line.251"></a>
+<span class="sourceLineNo">252</span> ls.append("\t").append(status.toString()).append("\n");<a name="line.252"></a>
+<span class="sourceLineNo">253</span> }<a name="line.253"></a>
+<span class="sourceLineNo">254</span> LOG.debug(Objects.toString(ls));<a name="line.254"></a>
+<span class="sourceLineNo">255</span> LOG.info("Splitting WALs out from under zombie. Expecting " + numWriters + " files.");<a name="line.255"></a>
+<span class="sourceLineNo">256</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf2, wals);<a name="line.256"></a>
+<span class="sourceLineNo">257</span> LOG.info("Finished splitting out from under zombie.");<a name="line.257"></a>
+<span class="sourceLineNo">258</span> Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);<a name="line.258"></a>
+<span class="sourceLineNo">259</span> assertEquals("wrong number of split files for region", numWriters, logfiles.length);<a name="line.259"></a>
+<span class="sourceLineNo">260</span> int count = 0;<a name="line.260"></a>
+<span class="sourceLineNo">261</span> for (Path logfile: logfiles) {<a name="line.261"></a>
+<span class="sourceLineNo">262</span> count += countWAL(logfile);<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span> return count;<a name="line.264"></a>
+<span class="sourceLineNo">265</span> }<a name="line.265"></a>
+<span class="sourceLineNo">266</span> });<a name="line.266"></a>
+<span class="sourceLineNo">267</span> LOG.info("zombie=" + counter.get() + ", robber=" + count);<a name="line.267"></a>
+<span class="sourceLineNo">268</span> assertTrue("The log file could have at most 1 extra log entry, but can't have less. " +<a name="line.268"></a>
+<span class="sourceLineNo">269</span> "Zombie could write " + counter.get() + " and logfile had only " + count,<a name="line.269"></a>
+<span class="sourceLineNo">270</span> counter.get() == count || counter.get() + 1 == count);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> } finally {<a name="line.271"></a>
+<span class="sourceLineNo">272</span> stop.set(true);<a name="line.272"></a>
+<span class="sourceLineNo">273</span> zombie.interrupt();<a name="line.273"></a>
+<span class="sourceLineNo">274</span> Threads.threadDumpingIsAlive(zombie);<a name="line.274"></a>
+<span class="sourceLineNo">275</span> }<a name="line.275"></a>
+<span class="sourceLineNo">276</span> }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span> /**<a name="line.278"></a>
+<span class="sourceLineNo">279</span> * This thread will keep writing to a 'wal' file even after the split process has started.<a name="line.279"></a>
+<span class="sourceLineNo">280</span> * It simulates a region server that was considered dead but woke up and wrote some more to the<a name="line.280"></a>
+<span class="sourceLineNo">281</span> * last log entry. Does its writing as an alternate user in another filesystem instance to<a name="line.281"></a>
+<span class="sourceLineNo">282</span> * simulate better it being a regionserver.<a name="line.282"></a>
+<span class="sourceLineNo">283</span> */<a name="line.283"></a>
+<span class="sourceLineNo">284</span> class ZombieLastLogWriterRegionServer extends Thread {<a name="line.284"></a>
+<span class="sourceLineNo">285</span> final AtomicLong editsCount;<a name="line.285"></a>
+<span class="sourceLineNo">286</span> final AtomicBoolean stop;<a name="line.286"></a>
+<span class="sourceLineNo">287</span> final int numOfWriters;<a name="line.287"></a>
+<span class="sourceLineNo">288</span> /**<a name="line.288"></a>
+<span class="sourceLineNo">289</span> * Region to write edits for.<a name="line.289"></a>
+<span class="sourceLineNo">290</span> */<a name="line.290"></a>
+<span class="sourceLineNo">291</span> final String region;<a name="line.291"></a>
+<span class="sourceLineNo">292</span> final User user;<a name="line.292"></a>
+<span class="sourceLineNo">293</span><a name="line.293"></a>
+<span class="sourceLineNo">294</span> public ZombieLastLogWriterRegionServer(AtomicLong counter, AtomicBoolean stop,<a name="line.294"></a>
+<span class="sourceLineNo">295</span> final String region, final int writers)<a name="line.295"></a>
+<span class="sourceLineNo">296</span> throws IOException, InterruptedException {<a name="line.296"></a>
+<span class="sourceLineNo">297</span> super("ZombieLastLogWriterRegionServer");<a name="line.297"></a>
+<span class="sourceLineNo">298</span> setDaemon(true);<a name="line.298"></a>
+<span class="sourceLineNo">299</span> this.stop = stop;<a name="line.299"></a>
+<span class="sourceLineNo">300</span> this.editsCount = counter;<a name="line.300"></a>
+<span class="sourceLineNo">301</span> this.region = region;<a name="line.301"></a>
+<span class="sourceLineNo">302</span> this.user = User.createUserForTesting(conf, ZOMBIE, GROUP);<a name="line.302"></a>
+<span class="sourceLineNo">303</span> numOfWriters = writers;<a name="line.303"></a>
+<span class="sourceLineNo">304</span> }<a name="line.304"></a>
+<span class="sourceLineNo">305</span><a name="line.305"></a>
+<span class="sourceLineNo">306</span> @Override<a name="line.306"></a>
+<span class="sourceLineNo">307</span> public void run() {<a name="line.307"></a>
+<span class="sourceLineNo">308</span> try {<a name="line.308"></a>
+<span class="sourceLineNo">309</span> doWriting();<a name="line.309"></a>
+<span class="sourceLineNo">310</span> } catch (IOException e) {<a name="line.310"></a>
+<span class="sourceLineNo">311</span> LOG.warn(getName() + " Writer exiting " + e);<a name="line.311"></a>
+<span class="sourceLineNo">312</span> } catch (InterruptedException e) {<a name="line.312"></a>
+<span class="sourceLineNo">313</span> LOG.warn(getName() + " Writer exiting " + e);<a name="line.313"></a>
+<span class="sourceLineNo">314</span> }<a name="line.314"></a>
+<span class="sourceLineNo">315</span> }<a name="line.315"></a>
+<span class="sourceLineNo">316</span><a name="line.316"></a>
+<span class="sourceLineNo">317</span> private void doWriting() throws IOException, InterruptedException {<a name="line.317"></a>
+<span class="sourceLineNo">318</span> this.user.runAs(new PrivilegedExceptionAction<Object>() {<a name="line.318"></a>
+<span class="sourceLineNo">319</span> @Override<a name="line.319"></a>
+<span class="sourceLineNo">320</span> public Object run() throws Exception {<a name="line.320"></a>
+<span class="sourceLineNo">321</span> // Index of the WAL we want to keep open. generateWALs will leave open the WAL whose<a name="line.321"></a>
+<span class="sourceLineNo">322</span> // index we supply here.<a name="line.322"></a>
+<span class="sourceLineNo">323</span> int walToKeepOpen = numOfWriters - 1;<a name="line.323"></a>
+<span class="sourceLineNo">324</span> // The below method writes numOfWriters files each with ENTRIES entries for a total of<a name="line.324"></a>
+<span class="sourceLineNo">325</span> // numOfWriters * ENTRIES added per column family in the region.<a name="line.325"></a>
+<span class="sourceLineNo">326</span> Writer writer = null;<a name="line.326"></a>
+<span class="sourceLineNo">327</span> try {<a name="line.327"></a>
+<span class="sourceLineNo">328</span> writer = generateWALs(numOfWriters, ENTRIES, walToKeepOpen);<a name="line.328"></a>
+<span class="sourceLineNo">329</span> } catch (IOException e1) {<a name="line.329"></a>
+<span class="sourceLineNo">330</span> throw new RuntimeException("Failed", e1);<a name="line.330"></a>
+<span class="sourceLineNo">331</span> }<a name="line.331"></a>
+<span class="sourceLineNo">332</span> // Update counter so has all edits written so far.<a name="line.332"></a>
+<span class="sourceLineNo">333</span> editsCount.addAndGet(numOfWriters * ENTRIES);<a name="line.333"></a>
+<span class="sourceLineNo">334</span> loop(writer);<a name="line.334"></a>
+<span class="sourceLineNo">335</span> // If we've been interruped, then things should have shifted out from under us.<a name="line.335"></a>
+<span class="sourceLineNo">336</span> // closing should error<a name="line.336"></a>
+<span class="sourceLineNo">337</span> try {<a name="line.337"></a>
+<span class="sourceLineNo">338</span> writer.close();<a name="line.338"></a>
+<span class="sourceLineNo">339</span> fail("Writing closing after parsing should give an error.");<a name="line.339"></a>
+<span class="sourceLineNo">340</span> } catch (IOException exception) {<a name="line.340"></a>
+<span class="sourceLineNo">341</span> LOG.debug("ignoring error when closing final writer.", exception);<a name="line.341"></a>
+<span class="sourceLineNo">342</span> }<a name="line.342"></a>
+<span class="sourceLineNo">343</span> return null;<a name="line.343"></a>
+<span class="sourceLineNo">344</span> }<a name="line.344"></a>
+<span class="sourceLineNo">345</span> });<a name="line.345"></a>
+<span class="sourceLineNo">346</span> }<a name="line.346"></a>
+<span class="sourceLineNo">347</span><a name="line.347"></a>
+<span class="sourceLineNo">348</span> private void loop(final Writer writer) {<a name="line.348"></a>
+<span class="sourceLineNo">349</span> byte [] regionBytes = Bytes.toBytes(this.region);<a name="line.349"></a>
+<span class="sourceLineNo">350</span> while (!stop.get()) {<a name="line.350"></a>
+<span class="sourceLineNo">351</span> try {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> long seq = appendEntry(writer, TABLE_NAME, regionBytes,<a name="line.352"></a>
+<span class="sourceLineNo">353</span> Bytes.toBytes("r" + editsCount.get()), regionBytes, QUALIFIER, VALUE, 0);<a name="line.353"></a>
+<span class="sourceLineNo">354</span> long count = editsCount.incrementAndGet();<a name="line.354"></a>
+<span class="sourceLineNo">355</span> LOG.info(getName() + " sync count=" + count + ", seq=" + seq);<a name="line.355"></a>
+<span class="sourceLineNo">356</span> try {<a name="line.356"></a>
+<span class="sourceLineNo">357</span> Thread.sleep(1);<a name="line.357"></a>
+<span class="sourceLineNo">358</span> } catch (InterruptedException e) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> //<a name="line.359"></a>
+<span class="sourceLineNo">360</span> }<a name="line.360"></a>
+<span class="sourceLineNo">361</span> } catch (IOException ex) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> LOG.error(getName() + " ex " + ex.toString());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> if (ex instanceof RemoteException) {<a name="line.363"></a>
+<span class="sourceLineNo">364</span> LOG.error("Juliet: got RemoteException " + ex.getMessage() +<a name="line.364"></a>
+<span class="sourceLineNo">365</span> " while writing " + (editsCount.get() + 1));<a name="line.365"></a>
+<span class="sourceLineNo">366</span> } else {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> LOG.error(getName() + " failed to write....at " + editsCount.get());<a name="line.367"></a>
+<span class="sourceLineNo">368</span> fail("Failed to write " + editsCount.get());<a name="line.368"></a>
+<span class="sourceLineNo">369</span> }<a name="line.369"></a>
+<span class="sourceLineNo">370</span> break;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> } catch (Throwable t) {<a name="line.371"></a>
+<span class="sourceLineNo">372</span> LOG.error(getName() + " HOW? " + t);<a name="line.372"></a>
+<span class="sourceLineNo">373</span> LOG.debug("exception details", t);<a name="line.373"></a>
+<span class="sourceLineNo">374</span> break;<a name="line.374"></a>
+<span class="sourceLineNo">375</span> }<a name="line.375"></a>
+<span class="sourceLineNo">376</span> }<a name="line.376"></a>
+<span class="sourceLineNo">377</span> LOG.info(getName() + " Writer exiting");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> }<a name="line.378"></a>
+<span class="sourceLineNo">379</span> }<a name="line.379"></a>
+<span class="sourceLineNo">380</span><a name="line.380"></a>
+<span class="sourceLineNo">381</span> /**<a name="line.381"></a>
+<span class="sourceLineNo">382</span> * {@see https://issues.apache.org/jira/browse/HBASE-3020}<a name="line.382"></a>
+<span class="sourceLineNo">383</span> */<a name="line.383"></a>
+<span class="sourceLineNo">384</span> @Test<a name="line.384"></a>
+<span class="sourceLineNo">385</span> public void testRecoveredEditsPathForMeta() throws IOException {<a name="line.385"></a>
+<span class="sourceLineNo">386</span> byte[] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes();<a name="line.386"></a>
+<span class="sourceLineNo">387</span> Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);<a name="line.387"></a>
+<span class="sourceLineNo">388</span> Path regiondir = new Path(tdir,<a name="line.388"></a>
+<span class="sourceLineNo">389</span> RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.389"></a>
+<span class="sourceLineNo">390</span> fs.mkdirs(regiondir);<a name="line.390"></a>
+<span class="sourceLineNo">391</span> long now = System.currentTimeMillis();<a name="line.391"></a>
+<span class="sourceLineNo">392</span> Entry entry =<a name="line.392"></a>
+<span class="sourceLineNo">393</span> new Entry(new WALKeyImpl(encoded,<a name="line.393"></a>
+<span class="sourceLineNo">394</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.394"></a>
+<span class="sourceLineNo">395</span> new WALEdit());<a name="line.395"></a>
+<span class="sourceLineNo">396</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.396"></a>
+<span class="sourceLineNo">397</span> FILENAME_BEING_SPLIT, TMPDIRNAME, conf);<a name="line.397"></a>
+<span class="sourceLineNo">398</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.399"></a>
+<span class="sourceLineNo">400</span> }<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span> /**<a name="line.402"></a>
+<span class="sourceLineNo">403</span> * Test old recovered edits file doesn't break WALSplitter.<a name="line.403"></a>
+<span class="sourceLineNo">404</span> * This is useful in upgrading old instances.<a name="line.404"></a>
+<span class="sourceLineNo">405</span> */<a name="line.405"></a>
+<span class="sourceLineNo">406</span> @Test<a name="line.406"></a>
+<span class="sourceLineNo">407</span> public void testOldRecoveredEditsFileSidelined() throws IOException {<a name="line.407"></a>
+<span class="sourceLineNo">408</span> byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes();<a name="line.408"></a>
+<span class="sourceLineNo">409</span> Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);<a name="line.409"></a>
+<span class="sourceLineNo">410</span> Path regiondir = new Path(tdir,<a name="line.410"></a>
+<span class="sourceLineNo">411</span> RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> fs.mkdirs(regiondir);<a name="line.412"></a>
+<span class="sourceLineNo">413</span> long now = System.currentTimeMillis();<a name="line.413"></a>
+<span class="sourceLineNo">414</span> Entry entry =<a name="line.414"></a>
+<span class="sourceLineNo">415</span> new Entry(new WALKeyImpl(encoded,<a name="line.415"></a>
+<span class="sourceLineNo">416</span> TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),<a name="line.416"></a>
+<span class="sourceLineNo">417</span> new WALEdit());<a name="line.417"></a>
+<span class="sourceLineNo">418</span> Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);<a name="line.418"></a>
+<span class="sourceLineNo">419</span> assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName());<a name="line.419"></a>
+<span class="sourceLineNo">420</span> fs.createNewFile(parent); // create a recovered.edits file<a name="line.420"></a>
+<span class="sourceLineNo">421</span><a name="line.421"></a>
+<span class="sourceLineNo">422</span> Path p = WALSplitter.getRegionSplitEditsPath(entry,<a name="line.422"></a>
+<span class="sourceLineNo">423</span> FILENAME_BEING_SPLIT, TMPDIRNAME, conf);<a name="line.423"></a>
+<span class="sourceLineNo">424</span> String parentOfParent = p.getParent().getParent().getName();<a name="line.424"></a>
+<span class="sourceLineNo">425</span> assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName());<a name="line.425"></a>
+<span class="sourceLineNo">426</span> WALFactory.createRecoveredEditsWriter(fs, p, conf).close();<a name="line.426"></a>
+<span class="sourceLineNo">427</span> }<a name="line.427"></a>
+<span class="sourceLineNo">428</span><a name="line.428"></a>
+<span class="sourceLineNo">429</span> private void useDifferentDFSClient() throws IOException {<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // make fs act as a different client now<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // initialize will create a new DFSClient with a new client ID<a name="line.431"></a>
+<span class="sourceLineNo">432</span> fs.initialize(fs.getUri(), conf);<a name="line.432"></a>
+<span class="sourceLineNo">433</span> }<a name="line.433"></a>
+<span class="sourceLineNo">434</span><a name="line.434"></a>
+<span class="sourceLineNo">435</span> @Test<a name="line.435"></a>
+<span class="sourceLineNo">436</span> public void testSplitPreservesEdits() throws IOException{<a name="line.436"></a>
+<span class="sourceLineNo">437</span> final String REGION = "region__1";<a name="line.437"></a>
+<span class="sourceLineNo">438</span> REGIONS.clear();<a name="line.438"></a>
+<span class="sourceLineNo">439</span> REGIONS.add(REGION);<a name="line.439"></a>
+<span class="sourceLineNo">440</span><a name="line.440"></a>
+<span class="sourceLineNo">441</span> generateWALs(1, 10, -1, 0);<a name="line.441"></a>
+<span class="sourceLineNo">442</span> useDifferentDFSClient();<a name="line.442"></a>
+<span class="sourceLineNo">443</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();<a name="line.444"></a>
+<span class="sourceLineNo">445</span> Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);<a name="line.445"></a>
+<span class="sourceLineNo">446</span> assertEquals(1, splitLog.length);<a name="line.446"></a>
<span class="sourceLineNo">447</span><a name="line.447"></a>
-<span class="sourceLineNo">448</span> @Test<a name="line.448"></a>
-<span class="sourceLineNo">449</span> public void testSplitRemovesRegionEventsEdits() throws IOException{<a name="line.449"></a>
-<span class="sourceLineNo">450</span> final String REGION = "region__1";<a name="line.450"></a>
-<span class="sourceLineNo">451</span> REGIONS.clear();<a name="line.451"></a>
-<span class="sourceLineNo">452</span> REGIONS.add(REGION);<a name="line.452"></a>
-<span class="sourceLineNo">453</span><a name="line.453"></a>
-<span class="sourceLineNo">454</span> generateWALs(1, 10, -1, 100);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> useDifferentDFSClient();<a name="line.455"></a>
-<span class="sourceLineNo">456</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.456"></a>
-<span class="sourceLineNo">457</span> Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();<a name="line.457"></a>
-<span class="sourceLineNo">458</span> Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);<a name="line.458"></a>
-<span class="sourceLineNo">459</span> assertEquals(1, splitLog.length);<a name="line.459"></a>
-<span class="sourceLineNo">460</span><a name="line.460"></a>
-<span class="sourceLineNo">461</span> assertFalse("edits differ after split", logsAreEqual(originalLog, splitLog[0]));<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // split log should only have the test edits<a name="line.462"></a>
-<span class="sourceLineNo">463</span> assertEquals(10, countWAL(splitLog[0]));<a name="line.463"></a>
-<span class="sourceLineNo">464</span> }<a name="line.464"></a>
-<span class="sourceLineNo">465</span><a name="line.465"></a>
-<span class="sourceLineNo">466</span><a name="line.466"></a>
-<span class="sourceLineNo">467</span> @Test<a name="line.467"></a>
-<span class="sourceLineNo">468</span> public void testSplitLeavesCompactionEventsEdits() throws IOException{<a name="line.468"></a>
-<span class="sourceLineNo">469</span> RegionInfo hri = RegionInfoBuilder.newBuilder(TABLE_NAME).build();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> REGIONS.clear();<a name="line.470"></a>
-<span class="sourceLineNo">471</span> REGIONS.add(hri.getEncodedName());<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Path regionDir = new Path(FSUtils.getTableDir(HBASEDIR, TABLE_NAME), hri.getEncodedName());<a name="line.472"></a>
-<span class="sourceLineNo">473</span> LOG.info("Creating region directory: " + regionDir);<a name="line.473"></a>
-<span class="sourceLineNo">474</span> assertTrue(fs.mkdirs(regionDir));<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Writer writer = generateWALs(1, 10, 0, 10);<a name="line.476"></a>
-<span class="sourceLineNo">477</span> String[] compactInputs = new String[]{"file1", "file2", "file3"};<a name="line.477"></a>
-<span class="sourceLineNo">478</span> String compactOutput = "file4";<a name="line.478"></a>
-<span class="sourceLineNo">479</span> appendCompactionEvent(writer, hri, compactInputs, compactOutput);<a name="line.479"></a>
-<span class="sourceLineNo">480</span> writer.close();<a name="line.480"></a>
-<span class="sourceLineNo">481</span><a name="line.481"></a>
-<span class="sourceLineNo">482</span> useDifferentDFSClient();<a name="line.482"></a>
-<span class="sourceLineNo">483</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.483"></a>
+<span class="sourceLineNo">448</span> assertTrue("edits differ after split", logsAreEqual(originalLog, splitLog[0]));<a name="line.448"></a>
+<span class="sourceLineNo">449</span> }<a name="line.449"></a>
+<span class="sourceLineNo">450</span><a name="line.450"></a>
+<span class="sourceLineNo">451</span> @Test<a name="line.451"></a>
+<span class="sourceLineNo">452</span> public void testSplitRemovesRegionEventsEdits() throws IOException{<a name="line.452"></a>
+<span class="sourceLineNo">453</span> final String REGION = "region__1";<a name="line.453"></a>
+<span class="sourceLineNo">454</span> REGIONS.clear();<a name="line.454"></a>
+<span class="sourceLineNo">455</span> REGIONS.add(REGION);<a name="line.455"></a>
+<span class="sourceLineNo">456</span><a name="line.456"></a>
+<span class="sourceLineNo">457</span> generateWALs(1, 10, -1, 100);<a name="line.457"></a>
+<span class="sourceLineNo">458</span> useDifferentDFSClient();<a name="line.458"></a>
+<span class="sourceLineNo">459</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.459"></a>
+<span class="sourceLineNo">460</span> Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();<a name="line.460"></a>
+<span class="sourceLineNo">461</span> Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);<a name="line.461"></a>
+<span class="sourceLineNo">462</span> assertEquals(1, splitLog.length);<a name="line.462"></a>
+<span class="sourceLineNo">463</span><a name="line.463"></a>
+<span class="sourceLineNo">464</span> assertFalse("edits differ after split", logsAreEqual(originalLog, splitLog[0]));<a name="line.464"></a>
+<span class="sourceLineNo">465</span> // split log should only have the test edits<a name="line.465"></a>
+<span class="sourceLineNo">466</span> assertEquals(10, countWAL(splitLog[0]));<a name="line.466"></a>
+<span class="sourceLineNo">467</span> }<a name="line.467"></a>
+<span class="sourceLineNo">468</span><a name="line.468"></a>
+<span class="sourceLineNo">469</span><a name="line.469"></a>
+<span class="sourceLineNo">470</span> @Test<a name="line.470"></a>
+<span class="sourceLineNo">471</span> public void testSplitLeavesCompactionEventsEdits() throws IOException{<a name="line.471"></a>
+<span class="sourceLineNo">472</span> RegionInfo hri = RegionInfoBuilder.newBuilder(TABLE_NAME).build();<a name="line.472"></a>
+<span class="sourceLineNo">473</span> REGIONS.clear();<a name="line.473"></a>
+<span class="sourceLineNo">474</span> REGIONS.add(hri.getEncodedName());<a name="line.474"></a>
+<span class="sourceLineNo">475</span> Path regionDir = new Path(FSUtils.getTableDir(HBASEDIR, TABLE_NAME), hri.getEncodedName());<a name="line.475"></a>
+<span class="sourceLineNo">476</span> LOG.info("Creating region directory: " + regionDir);<a name="line.476"></a>
+<span class="sourceLineNo">477</span> assertTrue(fs.mkdirs(regionDir));<a name="line.477"></a>
+<span class="sourceLineNo">478</span><a name="line.478"></a>
+<span class="sourceLineNo">479</span> Writer writer = generateWALs(1, 10, 0, 10);<a name="line.479"></a>
+<span class="sourceLineNo">480</span> String[] compactInputs = new String[]{"file1", "file2", "file3"};<a name="line.480"></a>
+<span class="sourceLineNo">481</span> String compactOutput = "file4";<a name="line.481"></a>
+<span class="sourceLineNo">482</span> appendCompactionEvent(writer, hri, compactInputs, compactOutput);<a name="line.482"></a>
+<span class="sourceLineNo">483</span> writer.close();<a name="line.483"></a>
<span class="sourceLineNo">484</span><a name="line.484"></a>
-<span class="sourceLineNo">485</span> Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();<a name="line.485"></a>
-<span class="sourceLineNo">486</span> // original log should have 10 test edits, 10 region markers, 1 compaction marker<a name="line.486"></a>
-<span class="sourceLineNo">487</span> assertEquals(21, countWAL(originalLog));<a name="line.487"></a>
-<span class="sourceLineNo">488</span><a name="line.488"></a>
-<span class="sourceLineNo">489</span> Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, hri.getEncodedName());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> assertEquals(1, splitLog.length);<a name="line.490"></a>
+<span class="sourceLineNo">485</span> useDifferentDFSClient();<a name="line.485"></a>
+<span class="sourceLineNo">486</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.486"></a>
+<span class="sourceLineNo">487</span><a name="line.487"></a>
+<span class="sourceLineNo">488</span> Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();<a name="line.488"></a>
+<span class="sourceLineNo">489</span> // original log should have 10 test edits, 10 region markers, 1 compaction marker<a name="line.489"></a>
+<span class="sourceLineNo">490</span> assertEquals(21, countWAL(originalLog));<a name="line.490"></a>
<span class="sourceLineNo">491</span><a name="line.491"></a>
-<span class="sourceLineNo">492</span> assertFalse("edits differ after split", logsAreEqual(originalLog, splitLog[0]));<a name="line.492"></a>
-<span class="sourceLineNo">493</span> // split log should have 10 test edits plus 1 compaction marker<a name="line.493"></a>
-<span class="sourceLineNo">494</span> assertEquals(11, countWAL(splitLog[0]));<a name="line.494"></a>
-<span class="sourceLineNo">495</span> }<a name="line.495"></a>
-<span class="sourceLineNo">496</span><a name="line.496"></a>
-<span class="sourceLineNo">497</span> /**<a name="line.497"></a>
-<span class="sourceLineNo">498</span> * @param expectedEntries -1 to not assert<a name="line.498"></a>
-<span class="sourceLineNo">499</span> * @return the count across all regions<a name="line.499"></a>
-<span class="sourceLineNo">500</span> */<a name="line.500"></a>
-<span class="sourceLineNo">501</span> private int splitAndCount(final int expectedFiles, final int expectedEntries)<a name="line.501"></a>
-<span class="sourceLineNo">502</span> throws IOException {<a name="line.502"></a>
-<span class="sourceLineNo">503</span> useDifferentDFSClient();<a name="line.503"></a>
-<span class="sourceLineNo">504</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.504"></a>
-<span class="sourceLineNo">505</span> int result = 0;<a name="line.505"></a>
-<span class="sourceLineNo">506</span> for (String region : REGIONS) {<a name="line.506"></a>
-<span class="sourceLineNo">507</span> Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);<a name="line.507"></a>
-<span class="sourceLineNo">508</span> assertEquals(expectedFiles, logfiles.length);<a name="line.508"></a>
-<span class="sourceLineNo">509</span> int count = 0;<a name="line.509"></a>
-<span class="sourceLineNo">510</span> for (Path logfile: logfiles) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> count += countWAL(logfile);<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> if (-1 != expectedEntries) {<a name="line.513"></a>
-<span class="sourceLineNo">514</span> assertEquals(expectedEntries, count);<a name="line.514"></a>
+<span class="sourceLineNo">492</span> Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, hri.getEncodedName());<a name="line.492"></a>
+<span class="sourceLineNo">493</span> assertEquals(1, splitLog.length);<a name="line.493"></a>
+<span class="sourceLineNo">494</span><a name="line.494"></a>
+<span class="sourceLineNo">495</span> assertFalse("edits differ after split", logsAreEqual(originalLog, splitLog[0]));<a name="line.495"></a>
+<span class="sourceLineNo">496</span> // split log should have 10 test edits plus 1 compaction marker<a name="line.496"></a>
+<span class="sourceLineNo">497</span> assertEquals(11, countWAL(splitLog[0]));<a name="line.497"></a>
+<span class="sourceLineNo">498</span> }<a name="line.498"></a>
+<span class="sourceLineNo">499</span><a name="line.499"></a>
+<span class="sourceLineNo">500</span> /**<a name="line.500"></a>
+<span class="sourceLineNo">501</span> * @param expectedEntries -1 to not assert<a name="line.501"></a>
+<span class="sourceLineNo">502</span> * @return the count across all regions<a name="line.502"></a>
+<span class="sourceLineNo">503</span> */<a name="line.503"></a>
+<span class="sourceLineNo">504</span> private int splitAndCount(final int expectedFiles, final int expectedEntries)<a name="line.504"></a>
+<span class="sourceLineNo">505</span> throws IOException {<a name="line.505"></a>
+<span class="sourceLineNo">506</span> useDifferentDFSClient();<a name="line.506"></a>
+<span class="sourceLineNo">507</span> WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> int result = 0;<a name="line.508"></a>
+<span class="sourceLineNo">509</span> for (String region : REGIONS) {<a name="line.509"></a>
+<span class="sourceLineNo">510</span> Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);<a name="line.510"></a>
+<span class="sourceLineNo">511</span> assertEquals(expectedFiles, logfiles.length);<a name="line.511"></a>
+<span class="sourceLineNo">512</span> int count = 0;<a name="line.512"></a>
+<span class="sourceLineNo">513</span> for (Path logfile: logfiles) {<a name="line.513"></a>
+<span class="sourceLineNo">514</span> count += countWAL(logfile);<a name="line.514"></a>
<span class="sourceLineNo">515</span> }<a name="line.515"></a>
-<span class="sourceLineNo">516</span> result += count;<a name="line.516"></a>
-<span class="sourceLineNo">517</span> }<a name="line.517"></a>
-<span class="sourceLineNo">518</span> return result;<a name="line.518"></a>
-<span class="sourceLineNo">519</span> }<a name="line.519"></a>
-<span class="sourceLineNo">520</span><a name="line.520"></a>
-<span class="sourceLineNo">521</span> @Test<a name="line.521"></a>
-<span class="sourceLineNo">522</span> public void testEmptyLogFiles() throws IOException {<a name="line.522"></a>
-<span class="sourceLineNo">523</span> testEmptyLogFiles(true);<a name="line.523"></a>
-<span class="sourceLineNo">524</span> }<a name="line.524"></a>
-<span class="sourceLineNo">525</span><a name="line.525"></a>
-<span class="sourceLineNo">526</span> @Test<a name="line.526"></a>
-<span class="sourceLineNo">527</span> public void testEmptyOpenLogFiles() throws IOException {<a name="line.527"></a>
-<span class="sourceLineNo">528</span> testEmptyLogFiles(false);<a name="line.528"></a>
-<span class="sourceLineNo">529</span> }<a name="line.529"></a>
-<span class="sourceLineNo">530</span><a name="line.530"></a>
-<span class="sourceLineNo">531</span> private void testEmptyLogFiles(final boolean close) throws IOException {<a name="line.531"></a>
-<span class="sourceLineNo">532</span> // we won't create the hlog dir until getWAL got called, so<a name="line.532"></a>
-<span class="sourceLineNo">533</span> // make dir here when testing empty log file<a name="line.533"></a>
-<span class="sourceLineNo">534</span> fs.mkdirs(WALDIR);<a name="line.534"></a>
-<span class="sourceLineNo">535</span> injectEmptyFile(".empty", close);<a name="line.535"></a>
-<span class="sourceLineNo">536</span> generateWALs(Integer.MAX_VALUE);<a name="line.536"></a>
-<span class="sourceLineNo">537</span> injectEmptyFile("empty", close);<a name="line.537"></a>
-<span class="sourceLineNo">538</span> splitAndCount(NUM_WRITERS, NUM_WRITERS * ENTRIES); // skip 2 empty<a name="line.538"></a>
-<span class="sourceLineNo">539</span> }<a name="line.539"></a>
-<span class="sourceLineNo">540</span><a name="line.540"></a>
-<span class="sourceLineNo">541</span> @Test<a name="line.541"></a>
-<span class="sourceLineNo">542</span> public void testOpenZeroLengthReportedFileButWithDataGetsSplit() throws IOException {<a name="line.542"></a>
-<span class="sourceLineNo">543</span> // generate logs but leave wal.dat.5 open.<a name="line.543"></a>
-<span class="sourceLineNo">544</span> generateWALs(5);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> splitAndCount(NUM_WRITERS, NUM_WRITERS * ENTRIES);<a name="line.545"></a>
-<span class="sourceLineNo">546</span> }<a name="line.546"></a>
-<span class="sourceLineNo">547</span><a name="line.547"></a>
-<span class="sourceLineNo">548</span> @Test<a name="line.548"></a>
-<span class="sourceLineNo">549</span> public void testTralingGarbageCorruptionFileSkipErrorsPasses() throws IOException {<a name="line.549"></a>
-<span class="sourceLineNo">550</span> conf.setBoolean(HBASE_SKIP_ERRORS, true);<a name="line.550"></a>
-<span class="sourceLineNo">551</span> generateWALs(Integer.MAX_VALUE);<a name="line.551"></a>
-<span class="sourceLineNo">552</span> corruptWAL(new Path(WALDIR, WAL_FILE_PREFIX + "5"),<a name="line.552"></a>
-<span class="sourceLineNo">553</span> Corruptions.APPEND_GARBAGE, true);<a name="line.553"></a>
-<span class="sourceLineNo">554</span> splitAndCount(NUM_WRITERS, NUM_WRITERS * ENTRIES);<a name="line.554"></a>
-<span class="sourceLineNo">555</span> }<a name="line.555"></a>
-<span class="sourceLineNo">556</span><a name="line.556"></a>
-<span class="sourceLineNo">557</span> @Test<a name="line.557"></a>
-<span class="sourceLineNo">558</span> public void testFirstLineCorruptionLogFileSkipErrorsPasses() throws IOException {<a name="line.558"></a>
-<span class="sourceLineNo">559</span> conf.setBoolean(HBASE_SKIP_ERRORS, true);<a name="line.559"></a>
-<span class="sourceLineNo">560</span> generateWALs(Integer.MAX_VALUE);<a name="line.560"></a>
-<span class="sourceLineNo">561</span> corruptWAL(new Path(WALDIR, WAL_FILE_PREFIX + "5"),<a name="line.561"></a>
-<span class="sourceLineNo">562</span> Corruptions.INSERT_GARBAGE_ON_FIRST_LINE, true);<a name="line.562"></a>
-<span class="sourceLineNo">563</span> splitAndCount(NUM_WRITERS - 1, (NUM_WRITERS - 1) * ENTRIES); //1 corrupt<a name="line.563"></a>
-<span class="sourceLineNo">564</span> }<a name="line.564"></a>
-<span class="sourceLineNo">565</span><a name="line.565"></a>
-<span class="sourceLineNo">566</span> @Test<a name="line.566"></a>
-<span class="sourceLineNo">567</span> public void testMiddleGarbageCorruptionSkipErrorsReadsHalfOfFile() throws IOException {<a name="line.567"></a>
-<span class="sourceLineNo">568</span> conf.setBoolean(HBASE_SKIP_ERRORS, true);<a name="line.568"></a>
-<span class="sourceLineNo">569</span> generateWALs(Integer.MAX_VALUE);<a name="line.569"></a>
-<span class="sourceLineNo">570</span> corruptWAL(new Path(WALDIR, WAL_FILE_PREFIX + "5"),<a name="line.570"></a>
-<span class="sourceLineNo">571</span> Corruptions.INSERT_GARBAGE_IN_THE_MIDDLE, false);<a name="line.571"></a>
-<span class="sourceLineNo">572</span> // the entries in the original logs are alternating regions<a name="line.572"></a>
-<span class="sourceLineNo">573</span> // considering the sequence file header, the middle corruption should<a name="line.573"></a>
-<span class="sourceLineNo">574</span> // affect at least half of the entries<a name="line.574"></a>
-<span class="sourceLineNo">575</span> int goodEntries = (NUM_WRITERS - 1) * ENTRIES;<a name="line.575"></a>
-<span class="sourceLineNo">576</span> int firstHalfEntries = (int) Math.ceil(ENTRIES / 2) - 1;<a name="line.576"></a>
-<span class="sourceLineNo">577</span> int allRegionsCount = splitAndCount(NUM_WRITERS, -1);<a name="line.577"></a>
-<span class="sourceLineNo">578</span> assertTrue("The file up to the corrupted area hasn't been parsed",<a name="line.578"></a>
-<span class="sourceLineNo">579</span> REGIONS.size() * (goodEntries + firstHalfEntries) <= allRegionsCount);<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span> @Test<a name="line.582"></a>
-<span class="sourceLineNo">583</span> public void testCorruptedFileGetsArchivedIfSkipErrors() throws IOException {<a name="line.583"></a>
-<span class="sourceLineNo">584</span> conf.setBoolean(HBASE_SKIP_ERRORS, true);<a name="line.584"></a>
-<span class="sourceLineNo">585</span> List<FaultyProtobufLogReader.FailureType> failureTypes = Arrays<a name="line.585"></a>
-<span class="sourceLineNo">586</span> .asList(FaultyProtobufLogReader.FailureType.values()).stream()<a name="line.586"></a>
-<span class="sourceLineNo">587</span> .filter(x -> x != FaultyProtobufLogReader.FailureType.NONE).collect(Collectors.toList());<a name="line.587"></a>
-<span class="sourceLineNo">588</span> for (FaultyProtobufLogReader.FailureType failureType : failureTypes) {<a name="line.588"></a>
-<span class="sourceLineNo">589</span> final Set<String> walDirContents = splitCorruptWALs(failureType);<a name="line.589"></a>
-<span class="sourceLineNo">590</span> final Set<String> archivedLogs = new HashSet<>();<a name="line.590"></a>
-<span class="sourceLineNo">591</span> final StringBuilder archived = new StringBuilder("Archived logs in CORRUPTDIR:");<a name="line.591"></a>
-<span class="sourceLineNo">592</span> for (FileStatus log : fs.listStatus(CORRUPTDIR)) {<a name="line.592"></a>
-<span class="sourceLineNo">593</span> archived.append("\n\t").append(log.toString());<a name="line.593"></a>
-<span class="sourceLineNo">594</span> archivedLogs.add(log.getPath().getName());<a name="line.594"></a>
-<span class="sourceLineNo">595</span> }<a name="line.595"></a>
-<span class="sourceLineNo">596</span> LOG.debug(archived.toString());<a name="line.596"></a>
-<span class="sourceLineNo">597</span> assertEquals(failureType.name() + ": expected to find all of our wals corrupt.", archivedLogs,<a name="line.597"></a>
-<span class="sourceLineNo">598</span> walDirContents);<a name="line.598"></a>
-<span class="sourceLineNo">599</span> }<a name="line.599"></a>
-<span class="sourceLineNo">600</span> }<a name="line.600"></a>
-<span class="sourceLineNo">601</span><a name="line.601"></a>
-<span class="sourceLineNo">602</span> /**<a name="line.602"></a>
-<span class="sourceLineNo">603</span> * @return set of wal names present prior to split attempt.<a name="line.603"></a>
-<span class="sourceLineNo">604</span> * @throws IOException if the split process fails<a name="line.604"></a>
-<span class="sourceLineNo">605</span> */<a name="line.605"></a>
-<span class="sourceLineNo">606</span> private Set<String> splitCorruptWALs(final FaultyProtobufLogReader.FailureType failureType)<a name="line.606"></a>
-<span class="sourceLineNo">607</span> throws IOException {<a name="line.607"></a>
-<span class="sourceLineNo">608</span> Class<?> backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl",<a name="line.608"></a>
-<span class="sourceLineNo">609</span> Reader.class);<a name="line.609"></a>
-<span class="sourceLineNo">610</span> InstrumentedLogWriter.activateFailure = false;<a name="line.610"></a>
-<span class="sourceLineNo">611</span><a name="line.611"></a>
-<span class="sourceLineNo">612</span> try {<a name="line.612"></a>
-<span class="sourceLineNo">613</span> conf.setClass("hbase.regionserver.hlog.reader.impl", FaultyProtobufLogReader.class,<a name="line.613"></a>
-<span class="sourceLineNo">614</span> Reader.class);<a name="line.614"></a>
-<span class="sourceLineNo">615</span> conf.set("faultyprotobuflogreader.failuretype", failureType.name());<a name="line.615"></a>
-<span class="sourceLineNo">616</span> // Clean up from previous tests or previous loop<a name="line.616"></a>
-<span class="sourceLineNo">617</span> try {<a name="line.617"></a>
-<span class="sourceLineNo">618</span> wals.shutdown();<a name="line.618"></a>
-<span class="sourceLineNo">619</span> } catch (IOException exception) {<a name="line.619"></a>
-<span class="sourceLineNo">620</span> // since we're splitting out from under the factory, we should expect some closing failures.<a name="line.620"></a>
-<span class="sourceLineNo">621</span> LOG.debug("Ignoring problem closing WALFactory.", exception);<a name="line.621"></a>
-<span class="sourceLineNo">622</span> }<a name="line.622"></a>
-<span class="sourceLineNo">623</span> wals.close();<a name="line.623"></a>
-<span class="sourceLineNo">624</span> try {<a name="line.624"></a>
-<span class="sourceLineNo">625</span> for (FileStatus log : fs.listStatus(CORRUPTDIR)) {<a name="line.625"></a>
-<span class="sourceLineNo">626</span> fs.delete(log.getPath(), true);<a name="line.626"></a>
-<span class="sourceLineNo">627</span> }<a name="line.627"></a>
-<span class="sourceLineNo">628</span> } catch (FileNotFoundException exception) {<a name="line.628"></a>
-<span class="sourceLineNo">629</span> LOG.debug("no previous CORRUPTDIR to clean.");<a name="line.629"></a>
-<span class="sourceLineNo">630</span> }<a name="line.630"></a>
-<span class="sourceLineNo">631</span> // change to the
<TRUNCATED>