You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/08/15 14:48:47 UTC
[01/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Repository: hbase-site
Updated Branches:
refs/heads/asf-site d682c43d8 -> 3b0a06ec3
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
index 380fb36..07ae748 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
@@ -117,93 +117,120 @@
<span class="sourceLineNo">109</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.109"></a>
<span class="sourceLineNo">110</span> }<a name="line.110"></a>
<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> @Test<a name="line.112"></a>
-<span class="sourceLineNo">113</span> public void testChildRollbackLoad() throws Exception {<a name="line.113"></a>
-<span class="sourceLineNo">114</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.114"></a>
-<span class="sourceLineNo">115</span> procEnv.triggerRollbackOnChild = true;<a name="line.115"></a>
-<span class="sourceLineNo">116</span><a name="line.116"></a>
-<span class="sourceLineNo">117</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.117"></a>
-<span class="sourceLineNo">118</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.119"></a>
-<span class="sourceLineNo">120</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.120"></a>
+<span class="sourceLineNo">112</span><a name="line.112"></a>
+<span class="sourceLineNo">113</span> /**<a name="line.113"></a>
+<span class="sourceLineNo">114</span> * Test the state setting that happens after store to WAL; in particular the bit where we<a name="line.114"></a>
+<span class="sourceLineNo">115</span> * set the parent runnable again after its children have all completed successfully.<a name="line.115"></a>
+<span class="sourceLineNo">116</span> * See HBASE-20978.<a name="line.116"></a>
+<span class="sourceLineNo">117</span> */<a name="line.117"></a>
+<span class="sourceLineNo">118</span> @Test<a name="line.118"></a>
+<span class="sourceLineNo">119</span> public void testChildLoadWithRestartAfterChildSuccess() throws Exception {<a name="line.119"></a>
+<span class="sourceLineNo">120</span> procEnv.toggleKillAfterStoreUpdate = true;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
-<span class="sourceLineNo">122</span> assertProcFailed(procId);<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> @Test<a name="line.125"></a>
-<span class="sourceLineNo">126</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.126"></a>
-<span class="sourceLineNo">127</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.127"></a>
-<span class="sourceLineNo">128</span> procEnv.triggerRollbackOnChild = true;<a name="line.128"></a>
-<span class="sourceLineNo">129</span><a name="line.129"></a>
-<span class="sourceLineNo">130</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.130"></a>
-<span class="sourceLineNo">131</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.131"></a>
-<span class="sourceLineNo">132</span> int restartCount = 0;<a name="line.132"></a>
-<span class="sourceLineNo">133</span> while (!procExecutor.isFinished(procId)) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.134"></a>
-<span class="sourceLineNo">135</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.135"></a>
-<span class="sourceLineNo">136</span> restartCount++;<a name="line.136"></a>
-<span class="sourceLineNo">137</span> }<a name="line.137"></a>
-<span class="sourceLineNo">138</span> assertEquals(2, restartCount);<a name="line.138"></a>
-<span class="sourceLineNo">139</span> assertProcFailed(procId);<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span> private void assertProcFailed(long procId) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.143"></a>
-<span class="sourceLineNo">144</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> assertEquals(true, result.isFailed());<a name="line.145"></a>
-<span class="sourceLineNo">146</span> LOG.info(result.getException().getMessage());<a name="line.146"></a>
-<span class="sourceLineNo">147</span> }<a name="line.147"></a>
-<span class="sourceLineNo">148</span><a name="line.148"></a>
-<span class="sourceLineNo">149</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.149"></a>
-<span class="sourceLineNo">150</span> public TestRootProcedure() {}<a name="line.150"></a>
-<span class="sourceLineNo">151</span><a name="line.151"></a>
-<span class="sourceLineNo">152</span> @Override<a name="line.152"></a>
-<span class="sourceLineNo">153</span> public Procedure[] execute(TestProcEnv env) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.157"></a>
-<span class="sourceLineNo">158</span> }<a name="line.158"></a>
-<span class="sourceLineNo">159</span><a name="line.159"></a>
-<span class="sourceLineNo">160</span> @Override<a name="line.160"></a>
-<span class="sourceLineNo">161</span> public void rollback(TestProcEnv env) {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> }<a name="line.162"></a>
-<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> @Override<a name="line.164"></a>
-<span class="sourceLineNo">165</span> public boolean abort(TestProcEnv env) {<a name="line.165"></a>
-<span class="sourceLineNo">166</span> return false;<a name="line.166"></a>
-<span class="sourceLineNo">167</span> }<a name="line.167"></a>
-<span class="sourceLineNo">168</span> }<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.170"></a>
-<span class="sourceLineNo">171</span> public TestChildProcedure() {}<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> @Override<a name="line.173"></a>
-<span class="sourceLineNo">174</span> public Procedure[] execute(TestProcEnv env) {<a name="line.174"></a>
-<span class="sourceLineNo">175</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.176"></a>
-<span class="sourceLineNo">177</span> }<a name="line.177"></a>
-<span class="sourceLineNo">178</span> if (env.triggerRollbackOnChild) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> setFailure("test", new Exception("test"));<a name="line.179"></a>
-<span class="sourceLineNo">180</span> }<a name="line.180"></a>
-<span class="sourceLineNo">181</span> return null;<a name="line.181"></a>
-<span class="sourceLineNo">182</span> }<a name="line.182"></a>
-<span class="sourceLineNo">183</span><a name="line.183"></a>
-<span class="sourceLineNo">184</span> @Override<a name="line.184"></a>
-<span class="sourceLineNo">185</span> public void rollback(TestProcEnv env) {<a name="line.185"></a>
-<span class="sourceLineNo">186</span> }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span> @Override<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public boolean abort(TestProcEnv env) {<a name="line.189"></a>
-<span class="sourceLineNo">190</span> return false;<a name="line.190"></a>
-<span class="sourceLineNo">191</span> }<a name="line.191"></a>
-<span class="sourceLineNo">192</span> }<a name="line.192"></a>
-<span class="sourceLineNo">193</span><a name="line.193"></a>
-<span class="sourceLineNo">194</span> private static class TestProcEnv {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.195"></a>
-<span class="sourceLineNo">196</span> public boolean triggerRollbackOnChild = false;<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>}<a name="line.198"></a>
+<span class="sourceLineNo">122</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.122"></a>
+<span class="sourceLineNo">123</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.123"></a>
+<span class="sourceLineNo">124</span> int restartCount = 0;<a name="line.124"></a>
+<span class="sourceLineNo">125</span> while (!procExecutor.isFinished(procId)) {<a name="line.125"></a>
+<span class="sourceLineNo">126</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.126"></a>
+<span class="sourceLineNo">127</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.127"></a>
+<span class="sourceLineNo">128</span> restartCount++;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span> assertEquals(4, restartCount);<a name="line.130"></a>
+<span class="sourceLineNo">131</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.131"></a>
+<span class="sourceLineNo">132</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.132"></a>
+<span class="sourceLineNo">133</span> }<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span> @Test<a name="line.135"></a>
+<span class="sourceLineNo">136</span> public void testChildRollbackLoad() throws Exception {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.137"></a>
+<span class="sourceLineNo">138</span> procEnv.triggerRollbackOnChild = true;<a name="line.138"></a>
+<span class="sourceLineNo">139</span><a name="line.139"></a>
+<span class="sourceLineNo">140</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.140"></a>
+<span class="sourceLineNo">141</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.141"></a>
+<span class="sourceLineNo">142</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span> assertProcFailed(procId);<a name="line.145"></a>
+<span class="sourceLineNo">146</span> }<a name="line.146"></a>
+<span class="sourceLineNo">147</span><a name="line.147"></a>
+<span class="sourceLineNo">148</span> @Test<a name="line.148"></a>
+<span class="sourceLineNo">149</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.149"></a>
+<span class="sourceLineNo">150</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.150"></a>
+<span class="sourceLineNo">151</span> procEnv.triggerRollbackOnChild = true;<a name="line.151"></a>
+<span class="sourceLineNo">152</span><a name="line.152"></a>
+<span class="sourceLineNo">153</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.153"></a>
+<span class="sourceLineNo">154</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.154"></a>
+<span class="sourceLineNo">155</span> int restartCount = 0;<a name="line.155"></a>
+<span class="sourceLineNo">156</span> while (!procExecutor.isFinished(procId)) {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.157"></a>
+<span class="sourceLineNo">158</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.158"></a>
+<span class="sourceLineNo">159</span> restartCount++;<a name="line.159"></a>
+<span class="sourceLineNo">160</span> }<a name="line.160"></a>
+<span class="sourceLineNo">161</span> assertEquals(2, restartCount);<a name="line.161"></a>
+<span class="sourceLineNo">162</span> assertProcFailed(procId);<a name="line.162"></a>
+<span class="sourceLineNo">163</span> }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span> private void assertProcFailed(long procId) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.166"></a>
+<span class="sourceLineNo">167</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.167"></a>
+<span class="sourceLineNo">168</span> assertEquals(true, result.isFailed());<a name="line.168"></a>
+<span class="sourceLineNo">169</span> LOG.info(result.getException().getMessage());<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> public TestRootProcedure() {}<a name="line.173"></a>
+<span class="sourceLineNo">174</span><a name="line.174"></a>
+<span class="sourceLineNo">175</span> @Override<a name="line.175"></a>
+<span class="sourceLineNo">176</span> public Procedure[] execute(TestProcEnv env) {<a name="line.176"></a>
+<span class="sourceLineNo">177</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.178"></a>
+<span class="sourceLineNo">179</span> }<a name="line.179"></a>
+<span class="sourceLineNo">180</span> if (env.toggleKillAfterStoreUpdate) {<a name="line.180"></a>
+<span class="sourceLineNo">181</span> ProcedureTestingUtility.toggleKillAfterStoreUpdate(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.183"></a>
+<span class="sourceLineNo">184</span> }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> @Override<a name="line.186"></a>
+<span class="sourceLineNo">187</span> public void rollback(TestProcEnv env) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> }<a name="line.188"></a>
+<span class="sourceLineNo">189</span><a name="line.189"></a>
+<span class="sourceLineNo">190</span> @Override<a name="line.190"></a>
+<span class="sourceLineNo">191</span> public boolean abort(TestProcEnv env) {<a name="line.191"></a>
+<span class="sourceLineNo">192</span> return false;<a name="line.192"></a>
+<span class="sourceLineNo">193</span> }<a name="line.193"></a>
+<span class="sourceLineNo">194</span> }<a name="line.194"></a>
+<span class="sourceLineNo">195</span><a name="line.195"></a>
+<span class="sourceLineNo">196</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> public TestChildProcedure() {}<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> @Override<a name="line.199"></a>
+<span class="sourceLineNo">200</span> public Procedure[] execute(TestProcEnv env) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.201"></a>
+<span class="sourceLineNo">202</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.202"></a>
+<span class="sourceLineNo">203</span> }<a name="line.203"></a>
+<span class="sourceLineNo">204</span> if (env.triggerRollbackOnChild) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span> setFailure("test", new Exception("test"));<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span> return null;<a name="line.207"></a>
+<span class="sourceLineNo">208</span> }<a name="line.208"></a>
+<span class="sourceLineNo">209</span><a name="line.209"></a>
+<span class="sourceLineNo">210</span> @Override<a name="line.210"></a>
+<span class="sourceLineNo">211</span> public void rollback(TestProcEnv env) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span> }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span> @Override<a name="line.214"></a>
+<span class="sourceLineNo">215</span> public boolean abort(TestProcEnv env) {<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return false;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span> }<a name="line.218"></a>
+<span class="sourceLineNo">219</span><a name="line.219"></a>
+<span class="sourceLineNo">220</span> private static class TestProcEnv {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> public boolean toggleKillAfterStoreUpdate = false;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> public boolean triggerRollbackOnChild = false;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> }<a name="line.224"></a>
+<span class="sourceLineNo">225</span>}<a name="line.225"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
index 380fb36..07ae748 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
@@ -117,93 +117,120 @@
<span class="sourceLineNo">109</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.109"></a>
<span class="sourceLineNo">110</span> }<a name="line.110"></a>
<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> @Test<a name="line.112"></a>
-<span class="sourceLineNo">113</span> public void testChildRollbackLoad() throws Exception {<a name="line.113"></a>
-<span class="sourceLineNo">114</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.114"></a>
-<span class="sourceLineNo">115</span> procEnv.triggerRollbackOnChild = true;<a name="line.115"></a>
-<span class="sourceLineNo">116</span><a name="line.116"></a>
-<span class="sourceLineNo">117</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.117"></a>
-<span class="sourceLineNo">118</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.119"></a>
-<span class="sourceLineNo">120</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.120"></a>
+<span class="sourceLineNo">112</span><a name="line.112"></a>
+<span class="sourceLineNo">113</span> /**<a name="line.113"></a>
+<span class="sourceLineNo">114</span> * Test the state setting that happens after store to WAL; in particular the bit where we<a name="line.114"></a>
+<span class="sourceLineNo">115</span> * set the parent runnable again after its children have all completed successfully.<a name="line.115"></a>
+<span class="sourceLineNo">116</span> * See HBASE-20978.<a name="line.116"></a>
+<span class="sourceLineNo">117</span> */<a name="line.117"></a>
+<span class="sourceLineNo">118</span> @Test<a name="line.118"></a>
+<span class="sourceLineNo">119</span> public void testChildLoadWithRestartAfterChildSuccess() throws Exception {<a name="line.119"></a>
+<span class="sourceLineNo">120</span> procEnv.toggleKillAfterStoreUpdate = true;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
-<span class="sourceLineNo">122</span> assertProcFailed(procId);<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> @Test<a name="line.125"></a>
-<span class="sourceLineNo">126</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.126"></a>
-<span class="sourceLineNo">127</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.127"></a>
-<span class="sourceLineNo">128</span> procEnv.triggerRollbackOnChild = true;<a name="line.128"></a>
-<span class="sourceLineNo">129</span><a name="line.129"></a>
-<span class="sourceLineNo">130</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.130"></a>
-<span class="sourceLineNo">131</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.131"></a>
-<span class="sourceLineNo">132</span> int restartCount = 0;<a name="line.132"></a>
-<span class="sourceLineNo">133</span> while (!procExecutor.isFinished(procId)) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.134"></a>
-<span class="sourceLineNo">135</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.135"></a>
-<span class="sourceLineNo">136</span> restartCount++;<a name="line.136"></a>
-<span class="sourceLineNo">137</span> }<a name="line.137"></a>
-<span class="sourceLineNo">138</span> assertEquals(2, restartCount);<a name="line.138"></a>
-<span class="sourceLineNo">139</span> assertProcFailed(procId);<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span> private void assertProcFailed(long procId) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.143"></a>
-<span class="sourceLineNo">144</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> assertEquals(true, result.isFailed());<a name="line.145"></a>
-<span class="sourceLineNo">146</span> LOG.info(result.getException().getMessage());<a name="line.146"></a>
-<span class="sourceLineNo">147</span> }<a name="line.147"></a>
-<span class="sourceLineNo">148</span><a name="line.148"></a>
-<span class="sourceLineNo">149</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.149"></a>
-<span class="sourceLineNo">150</span> public TestRootProcedure() {}<a name="line.150"></a>
-<span class="sourceLineNo">151</span><a name="line.151"></a>
-<span class="sourceLineNo">152</span> @Override<a name="line.152"></a>
-<span class="sourceLineNo">153</span> public Procedure[] execute(TestProcEnv env) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.157"></a>
-<span class="sourceLineNo">158</span> }<a name="line.158"></a>
-<span class="sourceLineNo">159</span><a name="line.159"></a>
-<span class="sourceLineNo">160</span> @Override<a name="line.160"></a>
-<span class="sourceLineNo">161</span> public void rollback(TestProcEnv env) {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> }<a name="line.162"></a>
-<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> @Override<a name="line.164"></a>
-<span class="sourceLineNo">165</span> public boolean abort(TestProcEnv env) {<a name="line.165"></a>
-<span class="sourceLineNo">166</span> return false;<a name="line.166"></a>
-<span class="sourceLineNo">167</span> }<a name="line.167"></a>
-<span class="sourceLineNo">168</span> }<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.170"></a>
-<span class="sourceLineNo">171</span> public TestChildProcedure() {}<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> @Override<a name="line.173"></a>
-<span class="sourceLineNo">174</span> public Procedure[] execute(TestProcEnv env) {<a name="line.174"></a>
-<span class="sourceLineNo">175</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.176"></a>
-<span class="sourceLineNo">177</span> }<a name="line.177"></a>
-<span class="sourceLineNo">178</span> if (env.triggerRollbackOnChild) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> setFailure("test", new Exception("test"));<a name="line.179"></a>
-<span class="sourceLineNo">180</span> }<a name="line.180"></a>
-<span class="sourceLineNo">181</span> return null;<a name="line.181"></a>
-<span class="sourceLineNo">182</span> }<a name="line.182"></a>
-<span class="sourceLineNo">183</span><a name="line.183"></a>
-<span class="sourceLineNo">184</span> @Override<a name="line.184"></a>
-<span class="sourceLineNo">185</span> public void rollback(TestProcEnv env) {<a name="line.185"></a>
-<span class="sourceLineNo">186</span> }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span> @Override<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public boolean abort(TestProcEnv env) {<a name="line.189"></a>
-<span class="sourceLineNo">190</span> return false;<a name="line.190"></a>
-<span class="sourceLineNo">191</span> }<a name="line.191"></a>
-<span class="sourceLineNo">192</span> }<a name="line.192"></a>
-<span class="sourceLineNo">193</span><a name="line.193"></a>
-<span class="sourceLineNo">194</span> private static class TestProcEnv {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.195"></a>
-<span class="sourceLineNo">196</span> public boolean triggerRollbackOnChild = false;<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>}<a name="line.198"></a>
+<span class="sourceLineNo">122</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.122"></a>
+<span class="sourceLineNo">123</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.123"></a>
+<span class="sourceLineNo">124</span> int restartCount = 0;<a name="line.124"></a>
+<span class="sourceLineNo">125</span> while (!procExecutor.isFinished(procId)) {<a name="line.125"></a>
+<span class="sourceLineNo">126</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.126"></a>
+<span class="sourceLineNo">127</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.127"></a>
+<span class="sourceLineNo">128</span> restartCount++;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span> assertEquals(4, restartCount);<a name="line.130"></a>
+<span class="sourceLineNo">131</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.131"></a>
+<span class="sourceLineNo">132</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.132"></a>
+<span class="sourceLineNo">133</span> }<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span> @Test<a name="line.135"></a>
+<span class="sourceLineNo">136</span> public void testChildRollbackLoad() throws Exception {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.137"></a>
+<span class="sourceLineNo">138</span> procEnv.triggerRollbackOnChild = true;<a name="line.138"></a>
+<span class="sourceLineNo">139</span><a name="line.139"></a>
+<span class="sourceLineNo">140</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.140"></a>
+<span class="sourceLineNo">141</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.141"></a>
+<span class="sourceLineNo">142</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span> assertProcFailed(procId);<a name="line.145"></a>
+<span class="sourceLineNo">146</span> }<a name="line.146"></a>
+<span class="sourceLineNo">147</span><a name="line.147"></a>
+<span class="sourceLineNo">148</span> @Test<a name="line.148"></a>
+<span class="sourceLineNo">149</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.149"></a>
+<span class="sourceLineNo">150</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.150"></a>
+<span class="sourceLineNo">151</span> procEnv.triggerRollbackOnChild = true;<a name="line.151"></a>
+<span class="sourceLineNo">152</span><a name="line.152"></a>
+<span class="sourceLineNo">153</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.153"></a>
+<span class="sourceLineNo">154</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.154"></a>
+<span class="sourceLineNo">155</span> int restartCount = 0;<a name="line.155"></a>
+<span class="sourceLineNo">156</span> while (!procExecutor.isFinished(procId)) {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.157"></a>
+<span class="sourceLineNo">158</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.158"></a>
+<span class="sourceLineNo">159</span> restartCount++;<a name="line.159"></a>
+<span class="sourceLineNo">160</span> }<a name="line.160"></a>
+<span class="sourceLineNo">161</span> assertEquals(2, restartCount);<a name="line.161"></a>
+<span class="sourceLineNo">162</span> assertProcFailed(procId);<a name="line.162"></a>
+<span class="sourceLineNo">163</span> }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span> private void assertProcFailed(long procId) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.166"></a>
+<span class="sourceLineNo">167</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.167"></a>
+<span class="sourceLineNo">168</span> assertEquals(true, result.isFailed());<a name="line.168"></a>
+<span class="sourceLineNo">169</span> LOG.info(result.getException().getMessage());<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> public TestRootProcedure() {}<a name="line.173"></a>
+<span class="sourceLineNo">174</span><a name="line.174"></a>
+<span class="sourceLineNo">175</span> @Override<a name="line.175"></a>
+<span class="sourceLineNo">176</span> public Procedure[] execute(TestProcEnv env) {<a name="line.176"></a>
+<span class="sourceLineNo">177</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.178"></a>
+<span class="sourceLineNo">179</span> }<a name="line.179"></a>
+<span class="sourceLineNo">180</span> if (env.toggleKillAfterStoreUpdate) {<a name="line.180"></a>
+<span class="sourceLineNo">181</span> ProcedureTestingUtility.toggleKillAfterStoreUpdate(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.183"></a>
+<span class="sourceLineNo">184</span> }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> @Override<a name="line.186"></a>
+<span class="sourceLineNo">187</span> public void rollback(TestProcEnv env) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> }<a name="line.188"></a>
+<span class="sourceLineNo">189</span><a name="line.189"></a>
+<span class="sourceLineNo">190</span> @Override<a name="line.190"></a>
+<span class="sourceLineNo">191</span> public boolean abort(TestProcEnv env) {<a name="line.191"></a>
+<span class="sourceLineNo">192</span> return false;<a name="line.192"></a>
+<span class="sourceLineNo">193</span> }<a name="line.193"></a>
+<span class="sourceLineNo">194</span> }<a name="line.194"></a>
+<span class="sourceLineNo">195</span><a name="line.195"></a>
+<span class="sourceLineNo">196</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> public TestChildProcedure() {}<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> @Override<a name="line.199"></a>
+<span class="sourceLineNo">200</span> public Procedure[] execute(TestProcEnv env) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.201"></a>
+<span class="sourceLineNo">202</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.202"></a>
+<span class="sourceLineNo">203</span> }<a name="line.203"></a>
+<span class="sourceLineNo">204</span> if (env.triggerRollbackOnChild) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span> setFailure("test", new Exception("test"));<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span> return null;<a name="line.207"></a>
+<span class="sourceLineNo">208</span> }<a name="line.208"></a>
+<span class="sourceLineNo">209</span><a name="line.209"></a>
+<span class="sourceLineNo">210</span> @Override<a name="line.210"></a>
+<span class="sourceLineNo">211</span> public void rollback(TestProcEnv env) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span> }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span> @Override<a name="line.214"></a>
+<span class="sourceLineNo">215</span> public boolean abort(TestProcEnv env) {<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return false;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span> }<a name="line.218"></a>
+<span class="sourceLineNo">219</span><a name="line.219"></a>
+<span class="sourceLineNo">220</span> private static class TestProcEnv {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> public boolean toggleKillAfterStoreUpdate = false;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> public boolean triggerRollbackOnChild = false;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> }<a name="line.224"></a>
+<span class="sourceLineNo">225</span>}<a name="line.225"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
index 380fb36..07ae748 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
@@ -117,93 +117,120 @@
<span class="sourceLineNo">109</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.109"></a>
<span class="sourceLineNo">110</span> }<a name="line.110"></a>
<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> @Test<a name="line.112"></a>
-<span class="sourceLineNo">113</span> public void testChildRollbackLoad() throws Exception {<a name="line.113"></a>
-<span class="sourceLineNo">114</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.114"></a>
-<span class="sourceLineNo">115</span> procEnv.triggerRollbackOnChild = true;<a name="line.115"></a>
-<span class="sourceLineNo">116</span><a name="line.116"></a>
-<span class="sourceLineNo">117</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.117"></a>
-<span class="sourceLineNo">118</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.119"></a>
-<span class="sourceLineNo">120</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.120"></a>
+<span class="sourceLineNo">112</span><a name="line.112"></a>
+<span class="sourceLineNo">113</span> /**<a name="line.113"></a>
+<span class="sourceLineNo">114</span> * Test the state setting that happens after store to WAL; in particular the bit where we<a name="line.114"></a>
+<span class="sourceLineNo">115</span> * set the parent runnable again after its children have all completed successfully.<a name="line.115"></a>
+<span class="sourceLineNo">116</span> * See HBASE-20978.<a name="line.116"></a>
+<span class="sourceLineNo">117</span> */<a name="line.117"></a>
+<span class="sourceLineNo">118</span> @Test<a name="line.118"></a>
+<span class="sourceLineNo">119</span> public void testChildLoadWithRestartAfterChildSuccess() throws Exception {<a name="line.119"></a>
+<span class="sourceLineNo">120</span> procEnv.toggleKillAfterStoreUpdate = true;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
-<span class="sourceLineNo">122</span> assertProcFailed(procId);<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> @Test<a name="line.125"></a>
-<span class="sourceLineNo">126</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.126"></a>
-<span class="sourceLineNo">127</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.127"></a>
-<span class="sourceLineNo">128</span> procEnv.triggerRollbackOnChild = true;<a name="line.128"></a>
-<span class="sourceLineNo">129</span><a name="line.129"></a>
-<span class="sourceLineNo">130</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.130"></a>
-<span class="sourceLineNo">131</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.131"></a>
-<span class="sourceLineNo">132</span> int restartCount = 0;<a name="line.132"></a>
-<span class="sourceLineNo">133</span> while (!procExecutor.isFinished(procId)) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.134"></a>
-<span class="sourceLineNo">135</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.135"></a>
-<span class="sourceLineNo">136</span> restartCount++;<a name="line.136"></a>
-<span class="sourceLineNo">137</span> }<a name="line.137"></a>
-<span class="sourceLineNo">138</span> assertEquals(2, restartCount);<a name="line.138"></a>
-<span class="sourceLineNo">139</span> assertProcFailed(procId);<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span> private void assertProcFailed(long procId) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.143"></a>
-<span class="sourceLineNo">144</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> assertEquals(true, result.isFailed());<a name="line.145"></a>
-<span class="sourceLineNo">146</span> LOG.info(result.getException().getMessage());<a name="line.146"></a>
-<span class="sourceLineNo">147</span> }<a name="line.147"></a>
-<span class="sourceLineNo">148</span><a name="line.148"></a>
-<span class="sourceLineNo">149</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.149"></a>
-<span class="sourceLineNo">150</span> public TestRootProcedure() {}<a name="line.150"></a>
-<span class="sourceLineNo">151</span><a name="line.151"></a>
-<span class="sourceLineNo">152</span> @Override<a name="line.152"></a>
-<span class="sourceLineNo">153</span> public Procedure[] execute(TestProcEnv env) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.157"></a>
-<span class="sourceLineNo">158</span> }<a name="line.158"></a>
-<span class="sourceLineNo">159</span><a name="line.159"></a>
-<span class="sourceLineNo">160</span> @Override<a name="line.160"></a>
-<span class="sourceLineNo">161</span> public void rollback(TestProcEnv env) {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> }<a name="line.162"></a>
-<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> @Override<a name="line.164"></a>
-<span class="sourceLineNo">165</span> public boolean abort(TestProcEnv env) {<a name="line.165"></a>
-<span class="sourceLineNo">166</span> return false;<a name="line.166"></a>
-<span class="sourceLineNo">167</span> }<a name="line.167"></a>
-<span class="sourceLineNo">168</span> }<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.170"></a>
-<span class="sourceLineNo">171</span> public TestChildProcedure() {}<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> @Override<a name="line.173"></a>
-<span class="sourceLineNo">174</span> public Procedure[] execute(TestProcEnv env) {<a name="line.174"></a>
-<span class="sourceLineNo">175</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.176"></a>
-<span class="sourceLineNo">177</span> }<a name="line.177"></a>
-<span class="sourceLineNo">178</span> if (env.triggerRollbackOnChild) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> setFailure("test", new Exception("test"));<a name="line.179"></a>
-<span class="sourceLineNo">180</span> }<a name="line.180"></a>
-<span class="sourceLineNo">181</span> return null;<a name="line.181"></a>
-<span class="sourceLineNo">182</span> }<a name="line.182"></a>
-<span class="sourceLineNo">183</span><a name="line.183"></a>
-<span class="sourceLineNo">184</span> @Override<a name="line.184"></a>
-<span class="sourceLineNo">185</span> public void rollback(TestProcEnv env) {<a name="line.185"></a>
-<span class="sourceLineNo">186</span> }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span> @Override<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public boolean abort(TestProcEnv env) {<a name="line.189"></a>
-<span class="sourceLineNo">190</span> return false;<a name="line.190"></a>
-<span class="sourceLineNo">191</span> }<a name="line.191"></a>
-<span class="sourceLineNo">192</span> }<a name="line.192"></a>
-<span class="sourceLineNo">193</span><a name="line.193"></a>
-<span class="sourceLineNo">194</span> private static class TestProcEnv {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.195"></a>
-<span class="sourceLineNo">196</span> public boolean triggerRollbackOnChild = false;<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>}<a name="line.198"></a>
+<span class="sourceLineNo">122</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.122"></a>
+<span class="sourceLineNo">123</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.123"></a>
+<span class="sourceLineNo">124</span> int restartCount = 0;<a name="line.124"></a>
+<span class="sourceLineNo">125</span> while (!procExecutor.isFinished(procId)) {<a name="line.125"></a>
+<span class="sourceLineNo">126</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.126"></a>
+<span class="sourceLineNo">127</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.127"></a>
+<span class="sourceLineNo">128</span> restartCount++;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span> assertEquals(4, restartCount);<a name="line.130"></a>
+<span class="sourceLineNo">131</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.131"></a>
+<span class="sourceLineNo">132</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.132"></a>
+<span class="sourceLineNo">133</span> }<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span> @Test<a name="line.135"></a>
+<span class="sourceLineNo">136</span> public void testChildRollbackLoad() throws Exception {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.137"></a>
+<span class="sourceLineNo">138</span> procEnv.triggerRollbackOnChild = true;<a name="line.138"></a>
+<span class="sourceLineNo">139</span><a name="line.139"></a>
+<span class="sourceLineNo">140</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.140"></a>
+<span class="sourceLineNo">141</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.141"></a>
+<span class="sourceLineNo">142</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span> assertProcFailed(procId);<a name="line.145"></a>
+<span class="sourceLineNo">146</span> }<a name="line.146"></a>
+<span class="sourceLineNo">147</span><a name="line.147"></a>
+<span class="sourceLineNo">148</span> @Test<a name="line.148"></a>
+<span class="sourceLineNo">149</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.149"></a>
+<span class="sourceLineNo">150</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.150"></a>
+<span class="sourceLineNo">151</span> procEnv.triggerRollbackOnChild = true;<a name="line.151"></a>
+<span class="sourceLineNo">152</span><a name="line.152"></a>
+<span class="sourceLineNo">153</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.153"></a>
+<span class="sourceLineNo">154</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.154"></a>
+<span class="sourceLineNo">155</span> int restartCount = 0;<a name="line.155"></a>
+<span class="sourceLineNo">156</span> while (!procExecutor.isFinished(procId)) {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.157"></a>
+<span class="sourceLineNo">158</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.158"></a>
+<span class="sourceLineNo">159</span> restartCount++;<a name="line.159"></a>
+<span class="sourceLineNo">160</span> }<a name="line.160"></a>
+<span class="sourceLineNo">161</span> assertEquals(2, restartCount);<a name="line.161"></a>
+<span class="sourceLineNo">162</span> assertProcFailed(procId);<a name="line.162"></a>
+<span class="sourceLineNo">163</span> }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span> private void assertProcFailed(long procId) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.166"></a>
+<span class="sourceLineNo">167</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.167"></a>
+<span class="sourceLineNo">168</span> assertEquals(true, result.isFailed());<a name="line.168"></a>
+<span class="sourceLineNo">169</span> LOG.info(result.getException().getMessage());<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> public TestRootProcedure() {}<a name="line.173"></a>
+<span class="sourceLineNo">174</span><a name="line.174"></a>
+<span class="sourceLineNo">175</span> @Override<a name="line.175"></a>
+<span class="sourceLineNo">176</span> public Procedure[] execute(TestProcEnv env) {<a name="line.176"></a>
+<span class="sourceLineNo">177</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.178"></a>
+<span class="sourceLineNo">179</span> }<a name="line.179"></a>
+<span class="sourceLineNo">180</span> if (env.toggleKillAfterStoreUpdate) {<a name="line.180"></a>
+<span class="sourceLineNo">181</span> ProcedureTestingUtility.toggleKillAfterStoreUpdate(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.183"></a>
+<span class="sourceLineNo">184</span> }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> @Override<a name="line.186"></a>
+<span class="sourceLineNo">187</span> public void rollback(TestProcEnv env) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> }<a name="line.188"></a>
+<span class="sourceLineNo">189</span><a name="line.189"></a>
+<span class="sourceLineNo">190</span> @Override<a name="line.190"></a>
+<span class="sourceLineNo">191</span> public boolean abort(TestProcEnv env) {<a name="line.191"></a>
+<span class="sourceLineNo">192</span> return false;<a name="line.192"></a>
+<span class="sourceLineNo">193</span> }<a name="line.193"></a>
+<span class="sourceLineNo">194</span> }<a name="line.194"></a>
+<span class="sourceLineNo">195</span><a name="line.195"></a>
+<span class="sourceLineNo">196</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> public TestChildProcedure() {}<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> @Override<a name="line.199"></a>
+<span class="sourceLineNo">200</span> public Procedure[] execute(TestProcEnv env) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.201"></a>
+<span class="sourceLineNo">202</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.202"></a>
+<span class="sourceLineNo">203</span> }<a name="line.203"></a>
+<span class="sourceLineNo">204</span> if (env.triggerRollbackOnChild) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span> setFailure("test", new Exception("test"));<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span> return null;<a name="line.207"></a>
+<span class="sourceLineNo">208</span> }<a name="line.208"></a>
+<span class="sourceLineNo">209</span><a name="line.209"></a>
+<span class="sourceLineNo">210</span> @Override<a name="line.210"></a>
+<span class="sourceLineNo">211</span> public void rollback(TestProcEnv env) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span> }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span> @Override<a name="line.214"></a>
+<span class="sourceLineNo">215</span> public boolean abort(TestProcEnv env) {<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return false;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span> }<a name="line.218"></a>
+<span class="sourceLineNo">219</span><a name="line.219"></a>
+<span class="sourceLineNo">220</span> private static class TestProcEnv {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> public boolean toggleKillAfterStoreUpdate = false;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> public boolean triggerRollbackOnChild = false;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> }<a name="line.224"></a>
+<span class="sourceLineNo">225</span>}<a name="line.225"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
index 380fb36..07ae748 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
@@ -117,93 +117,120 @@
<span class="sourceLineNo">109</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.109"></a>
<span class="sourceLineNo">110</span> }<a name="line.110"></a>
<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> @Test<a name="line.112"></a>
-<span class="sourceLineNo">113</span> public void testChildRollbackLoad() throws Exception {<a name="line.113"></a>
-<span class="sourceLineNo">114</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.114"></a>
-<span class="sourceLineNo">115</span> procEnv.triggerRollbackOnChild = true;<a name="line.115"></a>
-<span class="sourceLineNo">116</span><a name="line.116"></a>
-<span class="sourceLineNo">117</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.117"></a>
-<span class="sourceLineNo">118</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.118"></a>
-<span class="sourceLineNo">119</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.119"></a>
-<span class="sourceLineNo">120</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.120"></a>
+<span class="sourceLineNo">112</span><a name="line.112"></a>
+<span class="sourceLineNo">113</span> /**<a name="line.113"></a>
+<span class="sourceLineNo">114</span> * Test the state setting that happens after store to WAL; in particular the bit where we<a name="line.114"></a>
+<span class="sourceLineNo">115</span> * set the parent runnable again after its children have all completed successfully.<a name="line.115"></a>
+<span class="sourceLineNo">116</span> * See HBASE-20978.<a name="line.116"></a>
+<span class="sourceLineNo">117</span> */<a name="line.117"></a>
+<span class="sourceLineNo">118</span> @Test<a name="line.118"></a>
+<span class="sourceLineNo">119</span> public void testChildLoadWithRestartAfterChildSuccess() throws Exception {<a name="line.119"></a>
+<span class="sourceLineNo">120</span> procEnv.toggleKillAfterStoreUpdate = true;<a name="line.120"></a>
<span class="sourceLineNo">121</span><a name="line.121"></a>
-<span class="sourceLineNo">122</span> assertProcFailed(procId);<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> @Test<a name="line.125"></a>
-<span class="sourceLineNo">126</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.126"></a>
-<span class="sourceLineNo">127</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.127"></a>
-<span class="sourceLineNo">128</span> procEnv.triggerRollbackOnChild = true;<a name="line.128"></a>
-<span class="sourceLineNo">129</span><a name="line.129"></a>
-<span class="sourceLineNo">130</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.130"></a>
-<span class="sourceLineNo">131</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.131"></a>
-<span class="sourceLineNo">132</span> int restartCount = 0;<a name="line.132"></a>
-<span class="sourceLineNo">133</span> while (!procExecutor.isFinished(procId)) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.134"></a>
-<span class="sourceLineNo">135</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.135"></a>
-<span class="sourceLineNo">136</span> restartCount++;<a name="line.136"></a>
-<span class="sourceLineNo">137</span> }<a name="line.137"></a>
-<span class="sourceLineNo">138</span> assertEquals(2, restartCount);<a name="line.138"></a>
-<span class="sourceLineNo">139</span> assertProcFailed(procId);<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span> private void assertProcFailed(long procId) {<a name="line.142"></a>
-<span class="sourceLineNo">143</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.143"></a>
-<span class="sourceLineNo">144</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.144"></a>
-<span class="sourceLineNo">145</span> assertEquals(true, result.isFailed());<a name="line.145"></a>
-<span class="sourceLineNo">146</span> LOG.info(result.getException().getMessage());<a name="line.146"></a>
-<span class="sourceLineNo">147</span> }<a name="line.147"></a>
-<span class="sourceLineNo">148</span><a name="line.148"></a>
-<span class="sourceLineNo">149</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.149"></a>
-<span class="sourceLineNo">150</span> public TestRootProcedure() {}<a name="line.150"></a>
-<span class="sourceLineNo">151</span><a name="line.151"></a>
-<span class="sourceLineNo">152</span> @Override<a name="line.152"></a>
-<span class="sourceLineNo">153</span> public Procedure[] execute(TestProcEnv env) {<a name="line.153"></a>
-<span class="sourceLineNo">154</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.154"></a>
-<span class="sourceLineNo">155</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.155"></a>
-<span class="sourceLineNo">156</span> }<a name="line.156"></a>
-<span class="sourceLineNo">157</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.157"></a>
-<span class="sourceLineNo">158</span> }<a name="line.158"></a>
-<span class="sourceLineNo">159</span><a name="line.159"></a>
-<span class="sourceLineNo">160</span> @Override<a name="line.160"></a>
-<span class="sourceLineNo">161</span> public void rollback(TestProcEnv env) {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> }<a name="line.162"></a>
-<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> @Override<a name="line.164"></a>
-<span class="sourceLineNo">165</span> public boolean abort(TestProcEnv env) {<a name="line.165"></a>
-<span class="sourceLineNo">166</span> return false;<a name="line.166"></a>
-<span class="sourceLineNo">167</span> }<a name="line.167"></a>
-<span class="sourceLineNo">168</span> }<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.170"></a>
-<span class="sourceLineNo">171</span> public TestChildProcedure() {}<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> @Override<a name="line.173"></a>
-<span class="sourceLineNo">174</span> public Procedure[] execute(TestProcEnv env) {<a name="line.174"></a>
-<span class="sourceLineNo">175</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.175"></a>
-<span class="sourceLineNo">176</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.176"></a>
-<span class="sourceLineNo">177</span> }<a name="line.177"></a>
-<span class="sourceLineNo">178</span> if (env.triggerRollbackOnChild) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> setFailure("test", new Exception("test"));<a name="line.179"></a>
-<span class="sourceLineNo">180</span> }<a name="line.180"></a>
-<span class="sourceLineNo">181</span> return null;<a name="line.181"></a>
-<span class="sourceLineNo">182</span> }<a name="line.182"></a>
-<span class="sourceLineNo">183</span><a name="line.183"></a>
-<span class="sourceLineNo">184</span> @Override<a name="line.184"></a>
-<span class="sourceLineNo">185</span> public void rollback(TestProcEnv env) {<a name="line.185"></a>
-<span class="sourceLineNo">186</span> }<a name="line.186"></a>
-<span class="sourceLineNo">187</span><a name="line.187"></a>
-<span class="sourceLineNo">188</span> @Override<a name="line.188"></a>
-<span class="sourceLineNo">189</span> public boolean abort(TestProcEnv env) {<a name="line.189"></a>
-<span class="sourceLineNo">190</span> return false;<a name="line.190"></a>
-<span class="sourceLineNo">191</span> }<a name="line.191"></a>
-<span class="sourceLineNo">192</span> }<a name="line.192"></a>
-<span class="sourceLineNo">193</span><a name="line.193"></a>
-<span class="sourceLineNo">194</span> private static class TestProcEnv {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.195"></a>
-<span class="sourceLineNo">196</span> public boolean triggerRollbackOnChild = false;<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span>}<a name="line.198"></a>
+<span class="sourceLineNo">122</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.122"></a>
+<span class="sourceLineNo">123</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.123"></a>
+<span class="sourceLineNo">124</span> int restartCount = 0;<a name="line.124"></a>
+<span class="sourceLineNo">125</span> while (!procExecutor.isFinished(procId)) {<a name="line.125"></a>
+<span class="sourceLineNo">126</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.126"></a>
+<span class="sourceLineNo">127</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.127"></a>
+<span class="sourceLineNo">128</span> restartCount++;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span> assertEquals(4, restartCount);<a name="line.130"></a>
+<span class="sourceLineNo">131</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.131"></a>
+<span class="sourceLineNo">132</span> ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId);<a name="line.132"></a>
+<span class="sourceLineNo">133</span> }<a name="line.133"></a>
+<span class="sourceLineNo">134</span><a name="line.134"></a>
+<span class="sourceLineNo">135</span> @Test<a name="line.135"></a>
+<span class="sourceLineNo">136</span> public void testChildRollbackLoad() throws Exception {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> procEnv.toggleKillBeforeStoreUpdate = false;<a name="line.137"></a>
+<span class="sourceLineNo">138</span> procEnv.triggerRollbackOnChild = true;<a name="line.138"></a>
+<span class="sourceLineNo">139</span><a name="line.139"></a>
+<span class="sourceLineNo">140</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.140"></a>
+<span class="sourceLineNo">141</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.141"></a>
+<span class="sourceLineNo">142</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.143"></a>
+<span class="sourceLineNo">144</span><a name="line.144"></a>
+<span class="sourceLineNo">145</span> assertProcFailed(procId);<a name="line.145"></a>
+<span class="sourceLineNo">146</span> }<a name="line.146"></a>
+<span class="sourceLineNo">147</span><a name="line.147"></a>
+<span class="sourceLineNo">148</span> @Test<a name="line.148"></a>
+<span class="sourceLineNo">149</span> public void testChildRollbackLoadWithSteppedRestart() throws Exception {<a name="line.149"></a>
+<span class="sourceLineNo">150</span> procEnv.toggleKillBeforeStoreUpdate = true;<a name="line.150"></a>
+<span class="sourceLineNo">151</span> procEnv.triggerRollbackOnChild = true;<a name="line.151"></a>
+<span class="sourceLineNo">152</span><a name="line.152"></a>
+<span class="sourceLineNo">153</span> TestRootProcedure proc = new TestRootProcedure();<a name="line.153"></a>
+<span class="sourceLineNo">154</span> long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc);<a name="line.154"></a>
+<span class="sourceLineNo">155</span> int restartCount = 0;<a name="line.155"></a>
+<span class="sourceLineNo">156</span> while (!procExecutor.isFinished(procId)) {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> ProcedureTestingUtility.restart(procExecutor);<a name="line.157"></a>
+<span class="sourceLineNo">158</span> ProcedureTestingUtility.waitProcedure(procExecutor, proc);<a name="line.158"></a>
+<span class="sourceLineNo">159</span> restartCount++;<a name="line.159"></a>
+<span class="sourceLineNo">160</span> }<a name="line.160"></a>
+<span class="sourceLineNo">161</span> assertEquals(2, restartCount);<a name="line.161"></a>
+<span class="sourceLineNo">162</span> assertProcFailed(procId);<a name="line.162"></a>
+<span class="sourceLineNo">163</span> }<a name="line.163"></a>
+<span class="sourceLineNo">164</span><a name="line.164"></a>
+<span class="sourceLineNo">165</span> private void assertProcFailed(long procId) {<a name="line.165"></a>
+<span class="sourceLineNo">166</span> assertTrue("expected completed proc", procExecutor.isFinished(procId));<a name="line.166"></a>
+<span class="sourceLineNo">167</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.167"></a>
+<span class="sourceLineNo">168</span> assertEquals(true, result.isFailed());<a name="line.168"></a>
+<span class="sourceLineNo">169</span> LOG.info(result.getException().getMessage());<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public static class TestRootProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> public TestRootProcedure() {}<a name="line.173"></a>
+<span class="sourceLineNo">174</span><a name="line.174"></a>
+<span class="sourceLineNo">175</span> @Override<a name="line.175"></a>
+<span class="sourceLineNo">176</span> public Procedure[] execute(TestProcEnv env) {<a name="line.176"></a>
+<span class="sourceLineNo">177</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.177"></a>
+<span class="sourceLineNo">178</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.178"></a>
+<span class="sourceLineNo">179</span> }<a name="line.179"></a>
+<span class="sourceLineNo">180</span> if (env.toggleKillAfterStoreUpdate) {<a name="line.180"></a>
+<span class="sourceLineNo">181</span> ProcedureTestingUtility.toggleKillAfterStoreUpdate(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span> return new Procedure[] { new TestChildProcedure(), new TestChildProcedure() };<a name="line.183"></a>
+<span class="sourceLineNo">184</span> }<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> @Override<a name="line.186"></a>
+<span class="sourceLineNo">187</span> public void rollback(TestProcEnv env) {<a name="line.187"></a>
+<span class="sourceLineNo">188</span> }<a name="line.188"></a>
+<span class="sourceLineNo">189</span><a name="line.189"></a>
+<span class="sourceLineNo">190</span> @Override<a name="line.190"></a>
+<span class="sourceLineNo">191</span> public boolean abort(TestProcEnv env) {<a name="line.191"></a>
+<span class="sourceLineNo">192</span> return false;<a name="line.192"></a>
+<span class="sourceLineNo">193</span> }<a name="line.193"></a>
+<span class="sourceLineNo">194</span> }<a name="line.194"></a>
+<span class="sourceLineNo">195</span><a name="line.195"></a>
+<span class="sourceLineNo">196</span> public static class TestChildProcedure extends SequentialProcedure<TestProcEnv> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> public TestChildProcedure() {}<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> @Override<a name="line.199"></a>
+<span class="sourceLineNo">200</span> public Procedure[] execute(TestProcEnv env) {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> if (env.toggleKillBeforeStoreUpdate) {<a name="line.201"></a>
+<span class="sourceLineNo">202</span> ProcedureTestingUtility.toggleKillBeforeStoreUpdate(procExecutor);<a name="line.202"></a>
+<span class="sourceLineNo">203</span> }<a name="line.203"></a>
+<span class="sourceLineNo">204</span> if (env.triggerRollbackOnChild) {<a name="line.204"></a>
+<span class="sourceLineNo">205</span> setFailure("test", new Exception("test"));<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span> return null;<a name="line.207"></a>
+<span class="sourceLineNo">208</span> }<a name="line.208"></a>
+<span class="sourceLineNo">209</span><a name="line.209"></a>
+<span class="sourceLineNo">210</span> @Override<a name="line.210"></a>
+<span class="sourceLineNo">211</span> public void rollback(TestProcEnv env) {<a name="line.211"></a>
+<span class="sourceLineNo">212</span> }<a name="line.212"></a>
+<span class="sourceLineNo">213</span><a name="line.213"></a>
+<span class="sourceLineNo">214</span> @Override<a name="line.214"></a>
+<span class="sourceLineNo">215</span> public boolean abort(TestProcEnv env) {<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return false;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span> }<a name="line.218"></a>
+<span class="sourceLineNo">219</span><a name="line.219"></a>
+<span class="sourceLineNo">220</span> private static class TestProcEnv {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> public boolean toggleKillBeforeStoreUpdate = false;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> public boolean toggleKillAfterStoreUpdate = false;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> public boolean triggerRollbackOnChild = false;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> }<a name="line.224"></a>
+<span class="sourceLineNo">225</span>}<a name="line.225"></a>
[16/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span c
<TRUNCATED>
[14/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span class="sourceLineNo">600</span><a name="line.600"><
<TRUNCATED>
[04/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
index 45dd2e4..4310a16 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
@@ -175,399 +175,406 @@
<span class="sourceLineNo">167</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.167"></a>
<span class="sourceLineNo">168</span> }<a name="line.168"></a>
<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.170"></a>
-<span class="sourceLineNo">171</span> boolean value) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.172"></a>
-<span class="sourceLineNo">173</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.173"></a>
+<span class="sourceLineNo">170</span> public static <TEnv> void toggleKillAfterStoreUpdate(ProcedureExecutor<TEnv> procExecutor) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span> createExecutorTesting(procExecutor);<a name="line.171"></a>
+<span class="sourceLineNo">172</span> procExecutor.testing.killAfterStoreUpdate = !procExecutor.testing.killAfterStoreUpdate;<a name="line.172"></a>
+<span class="sourceLineNo">173</span> LOG.warn("Set Kill after store update to: " + procExecutor.testing.killAfterStoreUpdate);<a name="line.173"></a>
<span class="sourceLineNo">174</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.174"></a>
<span class="sourceLineNo">175</span> }<a name="line.175"></a>
<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.177"></a>
-<span class="sourceLineNo">178</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> if (procExecutor.testing == null) return;<a name="line.179"></a>
-<span class="sourceLineNo">180</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.180"></a>
-<span class="sourceLineNo">181</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.182"></a>
-<span class="sourceLineNo">183</span> 1, procExecutor.getCorePoolSize());<a name="line.183"></a>
-<span class="sourceLineNo">184</span> }<a name="line.184"></a>
-<span class="sourceLineNo">185</span> }<a name="line.185"></a>
-<span class="sourceLineNo">186</span><a name="line.186"></a>
-<span class="sourceLineNo">187</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.187"></a>
-<span class="sourceLineNo">188</span> throws IOException {<a name="line.188"></a>
-<span class="sourceLineNo">189</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.189"></a>
-<span class="sourceLineNo">190</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.190"></a>
-<span class="sourceLineNo">191</span> procStore.start(1);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.192"></a>
-<span class="sourceLineNo">193</span> try {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.194"></a>
-<span class="sourceLineNo">195</span> } finally {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> procStore.stop(false);<a name="line.196"></a>
-<span class="sourceLineNo">197</span> procExecutor.stop();<a name="line.197"></a>
-<span class="sourceLineNo">198</span> }<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> }<a name="line.203"></a>
-<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.205"></a>
-<span class="sourceLineNo">206</span> final long nonceGroup, final long nonce) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.207"></a>
-<span class="sourceLineNo">208</span> waitProcedure(procExecutor, procId);<a name="line.208"></a>
-<span class="sourceLineNo">209</span> return procId;<a name="line.209"></a>
+<span class="sourceLineNo">177</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.177"></a>
+<span class="sourceLineNo">178</span> boolean value) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.179"></a>
+<span class="sourceLineNo">180</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.180"></a>
+<span class="sourceLineNo">181</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span><a name="line.183"></a>
+<span class="sourceLineNo">184</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.184"></a>
+<span class="sourceLineNo">185</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.185"></a>
+<span class="sourceLineNo">186</span> if (procExecutor.testing == null) return;<a name="line.186"></a>
+<span class="sourceLineNo">187</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.187"></a>
+<span class="sourceLineNo">188</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.189"></a>
+<span class="sourceLineNo">190</span> 1, procExecutor.getCorePoolSize());<a name="line.190"></a>
+<span class="sourceLineNo">191</span> }<a name="line.191"></a>
+<span class="sourceLineNo">192</span> }<a name="line.192"></a>
+<span class="sourceLineNo">193</span><a name="line.193"></a>
+<span class="sourceLineNo">194</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.194"></a>
+<span class="sourceLineNo">195</span> throws IOException {<a name="line.195"></a>
+<span class="sourceLineNo">196</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.196"></a>
+<span class="sourceLineNo">197</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.197"></a>
+<span class="sourceLineNo">198</span> procStore.start(1);<a name="line.198"></a>
+<span class="sourceLineNo">199</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.199"></a>
+<span class="sourceLineNo">200</span> try {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.201"></a>
+<span class="sourceLineNo">202</span> } finally {<a name="line.202"></a>
+<span class="sourceLineNo">203</span> procStore.stop(false);<a name="line.203"></a>
+<span class="sourceLineNo">204</span> procExecutor.stop();<a name="line.204"></a>
+<span class="sourceLineNo">205</span> }<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.208"></a>
+<span class="sourceLineNo">209</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.209"></a>
<span class="sourceLineNo">210</span> }<a name="line.210"></a>
<span class="sourceLineNo">211</span><a name="line.211"></a>
-<span class="sourceLineNo">212</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
+<span class="sourceLineNo">212</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
<span class="sourceLineNo">213</span> final long nonceGroup, final long nonce) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.214"></a>
-<span class="sourceLineNo">215</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.215"></a>
-<span class="sourceLineNo">216</span> assertFalse(procId >= 0);<a name="line.216"></a>
-<span class="sourceLineNo">217</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.217"></a>
-<span class="sourceLineNo">218</span> }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.220"></a>
-<span class="sourceLineNo">221</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> Threads.sleepWithoutInterrupt(250);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> }<a name="line.223"></a>
-<span class="sourceLineNo">224</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.224"></a>
+<span class="sourceLineNo">214</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.214"></a>
+<span class="sourceLineNo">215</span> waitProcedure(procExecutor, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return procId;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span><a name="line.218"></a>
+<span class="sourceLineNo">219</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.219"></a>
+<span class="sourceLineNo">220</span> final long nonceGroup, final long nonce) {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.221"></a>
+<span class="sourceLineNo">222</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.222"></a>
+<span class="sourceLineNo">223</span> assertFalse(procId >= 0);<a name="line.223"></a>
+<span class="sourceLineNo">224</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.224"></a>
<span class="sourceLineNo">225</span> }<a name="line.225"></a>
<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.228"></a>
+<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.227"></a>
+<span class="sourceLineNo">228</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.228"></a>
<span class="sourceLineNo">229</span> Threads.sleepWithoutInterrupt(250);<a name="line.229"></a>
<span class="sourceLineNo">230</span> }<a name="line.230"></a>
-<span class="sourceLineNo">231</span> }<a name="line.231"></a>
-<span class="sourceLineNo">232</span><a name="line.232"></a>
-<span class="sourceLineNo">233</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.233"></a>
-<span class="sourceLineNo">234</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> waitProcedure(procExecutor, procIds[i]);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span><a name="line.238"></a>
-<span class="sourceLineNo">239</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.239"></a>
-<span class="sourceLineNo">240</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.240"></a>
-<span class="sourceLineNo">241</span> waitProcedure(procExecutor, procId);<a name="line.241"></a>
-<span class="sourceLineNo">242</span> }<a name="line.242"></a>
-<span class="sourceLineNo">243</span> }<a name="line.243"></a>
-<span class="sourceLineNo">244</span><a name="line.244"></a>
-<span class="sourceLineNo">245</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.245"></a>
-<span class="sourceLineNo">246</span> int stableRuns = 0;<a name="line.246"></a>
-<span class="sourceLineNo">247</span> while (stableRuns < 10) {<a name="line.247"></a>
-<span class="sourceLineNo">248</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span> stableRuns = 0;<a name="line.249"></a>
-<span class="sourceLineNo">250</span> Threads.sleepWithoutInterrupt(100);<a name="line.250"></a>
-<span class="sourceLineNo">251</span> } else {<a name="line.251"></a>
-<span class="sourceLineNo">252</span> stableRuns++;<a name="line.252"></a>
-<span class="sourceLineNo">253</span> Threads.sleepWithoutInterrupt(25);<a name="line.253"></a>
-<span class="sourceLineNo">254</span> }<a name="line.254"></a>
-<span class="sourceLineNo">255</span> }<a name="line.255"></a>
-<span class="sourceLineNo">256</span> }<a name="line.256"></a>
-<span class="sourceLineNo">257</span><a name="line.257"></a>
-<span class="sourceLineNo">258</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.258"></a>
-<span class="sourceLineNo">259</span> long procId) {<a name="line.259"></a>
-<span class="sourceLineNo">260</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.260"></a>
-<span class="sourceLineNo">261</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span><a name="line.263"></a>
-<span class="sourceLineNo">264</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.264"></a>
-<span class="sourceLineNo">265</span> long procId) {<a name="line.265"></a>
-<span class="sourceLineNo">266</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.266"></a>
-<span class="sourceLineNo">267</span> assertTrue("expected procedure result", result != null);<a name="line.267"></a>
-<span class="sourceLineNo">268</span> assertProcNotFailed(result);<a name="line.268"></a>
+<span class="sourceLineNo">231</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.231"></a>
+<span class="sourceLineNo">232</span> }<a name="line.232"></a>
+<span class="sourceLineNo">233</span><a name="line.233"></a>
+<span class="sourceLineNo">234</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.234"></a>
+<span class="sourceLineNo">235</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.235"></a>
+<span class="sourceLineNo">236</span> Threads.sleepWithoutInterrupt(250);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> }<a name="line.237"></a>
+<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">239</span><a name="line.239"></a>
+<span class="sourceLineNo">240</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.241"></a>
+<span class="sourceLineNo">242</span> waitProcedure(procExecutor, procIds[i]);<a name="line.242"></a>
+<span class="sourceLineNo">243</span> }<a name="line.243"></a>
+<span class="sourceLineNo">244</span> }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.247"></a>
+<span class="sourceLineNo">248</span> waitProcedure(procExecutor, procId);<a name="line.248"></a>
+<span class="sourceLineNo">249</span> }<a name="line.249"></a>
+<span class="sourceLineNo">250</span> }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> int stableRuns = 0;<a name="line.253"></a>
+<span class="sourceLineNo">254</span> while (stableRuns < 10) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.255"></a>
+<span class="sourceLineNo">256</span> stableRuns = 0;<a name="line.256"></a>
+<span class="sourceLineNo">257</span> Threads.sleepWithoutInterrupt(100);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> } else {<a name="line.258"></a>
+<span class="sourceLineNo">259</span> stableRuns++;<a name="line.259"></a>
+<span class="sourceLineNo">260</span> Threads.sleepWithoutInterrupt(25);<a name="line.260"></a>
+<span class="sourceLineNo">261</span> }<a name="line.261"></a>
+<span class="sourceLineNo">262</span> }<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.265"></a>
+<span class="sourceLineNo">266</span> long procId) {<a name="line.266"></a>
+<span class="sourceLineNo">267</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.267"></a>
+<span class="sourceLineNo">268</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.268"></a>
<span class="sourceLineNo">269</span> }<a name="line.269"></a>
<span class="sourceLineNo">270</span><a name="line.270"></a>
-<span class="sourceLineNo">271</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.271"></a>
-<span class="sourceLineNo">272</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.272"></a>
-<span class="sourceLineNo">273</span> }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.275"></a>
-<span class="sourceLineNo">276</span> final long procId) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.277"></a>
-<span class="sourceLineNo">278</span> assertTrue("expected procedure result", result != null);<a name="line.278"></a>
-<span class="sourceLineNo">279</span> return assertProcFailed(result);<a name="line.279"></a>
+<span class="sourceLineNo">271</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.271"></a>
+<span class="sourceLineNo">272</span> long procId) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.273"></a>
+<span class="sourceLineNo">274</span> assertTrue("expected procedure result", result != null);<a name="line.274"></a>
+<span class="sourceLineNo">275</span> assertProcNotFailed(result);<a name="line.275"></a>
+<span class="sourceLineNo">276</span> }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.278"></a>
+<span class="sourceLineNo">279</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.279"></a>
<span class="sourceLineNo">280</span> }<a name="line.280"></a>
<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.282"></a>
-<span class="sourceLineNo">283</span> assertEquals(true, result.isFailed());<a name="line.283"></a>
-<span class="sourceLineNo">284</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.284"></a>
-<span class="sourceLineNo">285</span> return getExceptionCause(result);<a name="line.285"></a>
-<span class="sourceLineNo">286</span> }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.288"></a>
-<span class="sourceLineNo">289</span> Throwable cause = assertProcFailed(result);<a name="line.289"></a>
-<span class="sourceLineNo">290</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.290"></a>
-<span class="sourceLineNo">291</span> }<a name="line.291"></a>
-<span class="sourceLineNo">292</span><a name="line.292"></a>
-<span class="sourceLineNo">293</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> Throwable cause = assertProcFailed(result);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.298"></a>
-<span class="sourceLineNo">299</span> Throwable cause = assertProcFailed(result);<a name="line.299"></a>
-<span class="sourceLineNo">300</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.300"></a>
-<span class="sourceLineNo">301</span> cause instanceof IllegalArgumentIOException);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> }<a name="line.302"></a>
-<span class="sourceLineNo">303</span><a name="line.303"></a>
-<span class="sourceLineNo">304</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.304"></a>
-<span class="sourceLineNo">305</span> assert procInfo.isFailed();<a name="line.305"></a>
-<span class="sourceLineNo">306</span> Throwable cause = procInfo.getException().getCause();<a name="line.306"></a>
-<span class="sourceLineNo">307</span> return cause == null ? procInfo.getException() : cause;<a name="line.307"></a>
-<span class="sourceLineNo">308</span> }<a name="line.308"></a>
-<span class="sourceLineNo">309</span><a name="line.309"></a>
-<span class="sourceLineNo">310</span> /**<a name="line.310"></a>
-<span class="sourceLineNo">311</span> * Run through all procedure flow states TWICE while also restarting<a name="line.311"></a>
-<span class="sourceLineNo">312</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.312"></a>
-<span class="sourceLineNo">313</span> *<a name="line.313"></a>
-<span class="sourceLineNo">314</span> *<p>It does<a name="line.314"></a>
-<span class="sourceLineNo">315</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.315"></a>
-<span class="sourceLineNo">316</span> * <li>Restart executor/store<a name="line.316"></a>
-<span class="sourceLineNo">317</span> * <li>Execute step N - and then save to store<a name="line.317"></a>
-<span class="sourceLineNo">318</span> * </ol><a name="line.318"></a>
-<span class="sourceLineNo">319</span> *<a name="line.319"></a>
-<span class="sourceLineNo">320</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.320"></a>
-<span class="sourceLineNo">321</span> * idempotent.<a name="line.321"></a>
-<span class="sourceLineNo">322</span> */<a name="line.322"></a>
-<span class="sourceLineNo">323</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.323"></a>
-<span class="sourceLineNo">324</span> final long procId) throws Exception {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.325"></a>
-<span class="sourceLineNo">326</span> }<a name="line.326"></a>
-<span class="sourceLineNo">327</span><a name="line.327"></a>
-<span class="sourceLineNo">328</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.328"></a>
-<span class="sourceLineNo">329</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.329"></a>
-<span class="sourceLineNo">330</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.330"></a>
-<span class="sourceLineNo">331</span> }<a name="line.331"></a>
-<span class="sourceLineNo">332</span><a name="line.332"></a>
-<span class="sourceLineNo">333</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.333"></a>
-<span class="sourceLineNo">334</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.334"></a>
-<span class="sourceLineNo">335</span> throws Exception {<a name="line.335"></a>
-<span class="sourceLineNo">336</span> Procedure proc = procExec.getProcedure(procId);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> waitProcedure(procExec, procId);<a name="line.337"></a>
-<span class="sourceLineNo">338</span> assertEquals(false, procExec.isRunning());<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> proc = procExec.getProcedure(procId);<a name="line.340"></a>
-<span class="sourceLineNo">341</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> if (customRestart != null) {<a name="line.342"></a>
-<span class="sourceLineNo">343</span> customRestart.run();<a name="line.343"></a>
-<span class="sourceLineNo">344</span> } else {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> restart(procExec);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> }<a name="line.346"></a>
-<span class="sourceLineNo">347</span> waitProcedure(procExec, procId);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span><a name="line.349"></a>
-<span class="sourceLineNo">350</span> assertEquals(true, procExec.isRunning());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> if (expectFailure) {<a name="line.351"></a>
-<span class="sourceLineNo">352</span> assertProcFailed(procExec, procId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> } else {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> assertProcNotFailed(procExec, procId);<a name="line.354"></a>
+<span class="sourceLineNo">282</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.282"></a>
+<span class="sourceLineNo">283</span> final long procId) {<a name="line.283"></a>
+<span class="sourceLineNo">284</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.284"></a>
+<span class="sourceLineNo">285</span> assertTrue("expected procedure result", result != null);<a name="line.285"></a>
+<span class="sourceLineNo">286</span> return assertProcFailed(result);<a name="line.286"></a>
+<span class="sourceLineNo">287</span> }<a name="line.287"></a>
+<span class="sourceLineNo">288</span><a name="line.288"></a>
+<span class="sourceLineNo">289</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.289"></a>
+<span class="sourceLineNo">290</span> assertEquals(true, result.isFailed());<a name="line.290"></a>
+<span class="sourceLineNo">291</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.291"></a>
+<span class="sourceLineNo">292</span> return getExceptionCause(result);<a name="line.292"></a>
+<span class="sourceLineNo">293</span> }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span> Throwable cause = assertProcFailed(result);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.297"></a>
+<span class="sourceLineNo">298</span> }<a name="line.298"></a>
+<span class="sourceLineNo">299</span><a name="line.299"></a>
+<span class="sourceLineNo">300</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.300"></a>
+<span class="sourceLineNo">301</span> Throwable cause = assertProcFailed(result);<a name="line.301"></a>
+<span class="sourceLineNo">302</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.302"></a>
+<span class="sourceLineNo">303</span> }<a name="line.303"></a>
+<span class="sourceLineNo">304</span><a name="line.304"></a>
+<span class="sourceLineNo">305</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.305"></a>
+<span class="sourceLineNo">306</span> Throwable cause = assertProcFailed(result);<a name="line.306"></a>
+<span class="sourceLineNo">307</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.307"></a>
+<span class="sourceLineNo">308</span> cause instanceof IllegalArgumentIOException);<a name="line.308"></a>
+<span class="sourceLineNo">309</span> }<a name="line.309"></a>
+<span class="sourceLineNo">310</span><a name="line.310"></a>
+<span class="sourceLineNo">311</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span> assert procInfo.isFailed();<a name="line.312"></a>
+<span class="sourceLineNo">313</span> Throwable cause = procInfo.getException().getCause();<a name="line.313"></a>
+<span class="sourceLineNo">314</span> return cause == null ? procInfo.getException() : cause;<a name="line.314"></a>
+<span class="sourceLineNo">315</span> }<a name="line.315"></a>
+<span class="sourceLineNo">316</span><a name="line.316"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Run through all procedure flow states TWICE while also restarting<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.319"></a>
+<span class="sourceLineNo">320</span> *<a name="line.320"></a>
+<span class="sourceLineNo">321</span> *<p>It does<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Restart executor/store<a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Execute step N - and then save to store<a name="line.324"></a>
+<span class="sourceLineNo">325</span> * </ol><a name="line.325"></a>
+<span class="sourceLineNo">326</span> *<a name="line.326"></a>
+<span class="sourceLineNo">327</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * idempotent.<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.330"></a>
+<span class="sourceLineNo">331</span> final long procId) throws Exception {<a name="line.331"></a>
+<span class="sourceLineNo">332</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.332"></a>
+<span class="sourceLineNo">333</span> }<a name="line.333"></a>
+<span class="sourceLineNo">334</span><a name="line.334"></a>
+<span class="sourceLineNo">335</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.335"></a>
+<span class="sourceLineNo">336</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.337"></a>
+<span class="sourceLineNo">338</span> }<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.340"></a>
+<span class="sourceLineNo">341</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.341"></a>
+<span class="sourceLineNo">342</span> throws Exception {<a name="line.342"></a>
+<span class="sourceLineNo">343</span> Procedure proc = procExec.getProcedure(procId);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> waitProcedure(procExec, procId);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> assertEquals(false, procExec.isRunning());<a name="line.345"></a>
+<span class="sourceLineNo">346</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.346"></a>
+<span class="sourceLineNo">347</span> proc = procExec.getProcedure(procId);<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.348"></a>
+<span class="sourceLineNo">349</span> if (customRestart != null) {<a name="line.349"></a>
+<span class="sourceLineNo">350</span> customRestart.run();<a name="line.350"></a>
+<span class="sourceLineNo">351</span> } else {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> restart(procExec);<a name="line.352"></a>
+<span class="sourceLineNo">353</span> }<a name="line.353"></a>
+<span class="sourceLineNo">354</span> waitProcedure(procExec, procId);<a name="line.354"></a>
<span class="sourceLineNo">355</span> }<a name="line.355"></a>
-<span class="sourceLineNo">356</span> }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.358"></a>
-<span class="sourceLineNo">359</span> public NoopProcedure() {}<a name="line.359"></a>
-<span class="sourceLineNo">360</span><a name="line.360"></a>
-<span class="sourceLineNo">361</span> @Override<a name="line.361"></a>
-<span class="sourceLineNo">362</span> protected Procedure[] execute(TEnv env)<a name="line.362"></a>
-<span class="sourceLineNo">363</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> return null;<a name="line.364"></a>
-<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span> @Override<a name="line.367"></a>
-<span class="sourceLineNo">368</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> @Override<a name="line.371"></a>
-<span class="sourceLineNo">372</span> protected boolean abort(TEnv env) { return false; }<a name="line.372"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span> assertEquals(true, procExec.isRunning());<a name="line.357"></a>
+<span class="sourceLineNo">358</span> if (expectFailure) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> assertProcFailed(procExec, procId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> } else {<a name="line.360"></a>
+<span class="sourceLineNo">361</span> assertProcNotFailed(procExec, procId);<a name="line.361"></a>
+<span class="sourceLineNo">362</span> }<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.365"></a>
+<span class="sourceLineNo">366</span> public NoopProcedure() {}<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span> @Override<a name="line.368"></a>
+<span class="sourceLineNo">369</span> protected Procedure[] execute(TEnv env)<a name="line.369"></a>
+<span class="sourceLineNo">370</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.370"></a>
+<span class="sourceLineNo">371</span> return null;<a name="line.371"></a>
+<span class="sourceLineNo">372</span> }<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
<span class="sourceLineNo">374</span> @Override<a name="line.374"></a>
-<span class="sourceLineNo">375</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.375"></a>
-<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> @Override<a name="line.379"></a>
-<span class="sourceLineNo">380</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.380"></a>
-<span class="sourceLineNo">381</span> throws IOException {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.385"></a>
-<span class="sourceLineNo">386</span> private byte[] data = null;<a name="line.386"></a>
-<span class="sourceLineNo">387</span><a name="line.387"></a>
-<span class="sourceLineNo">388</span> public TestProcedure() {}<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> public TestProcedure(long procId) {<a name="line.390"></a>
-<span class="sourceLineNo">391</span> this(procId, 0);<a name="line.391"></a>
-<span class="sourceLineNo">392</span> }<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span> public TestProcedure(long procId, long parentId) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span> this(procId, parentId, null);<a name="line.395"></a>
-<span class="sourceLineNo">396</span> }<a name="line.396"></a>
-<span class="sourceLineNo">397</span><a name="line.397"></a>
-<span class="sourceLineNo">398</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> this(procId, parentId, parentId, data);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> }<a name="line.400"></a>
-<span class="sourceLineNo">401</span><a name="line.401"></a>
-<span class="sourceLineNo">402</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> setData(data);<a name="line.403"></a>
-<span class="sourceLineNo">404</span> setProcId(procId);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> if (parentId > 0) {<a name="line.405"></a>
-<span class="sourceLineNo">406</span> setParentProcId(parentId);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> }<a name="line.407"></a>
-<span class="sourceLineNo">408</span> if (rootId > 0 || parentId > 0) {<a name="line.408"></a>
-<span class="sourceLineNo">409</span> setRootProcId(rootId);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> }<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span><a name="line.412"></a>
-<span class="sourceLineNo">413</span> public void addStackId(final int index) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span> addStackIndex(index);<a name="line.414"></a>
-<span class="sourceLineNo">415</span> }<a name="line.415"></a>
-<span class="sourceLineNo">416</span><a name="line.416"></a>
-<span class="sourceLineNo">417</span> public void setSuccessState() {<a name="line.417"></a>
-<span class="sourceLineNo">418</span> setState(ProcedureState.SUCCESS);<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span><a name="line.420"></a>
-<span class="sourceLineNo">421</span> public void setData(final byte[] data) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> this.data = data;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span><a name="line.424"></a>
-<span class="sourceLineNo">425</span> @Override<a name="line.425"></a>
-<span class="sourceLineNo">426</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.426"></a>
-<span class="sourceLineNo">427</span> throws IOException {<a name="line.427"></a>
-<span class="sourceLineNo">428</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.428"></a>
-<span class="sourceLineNo">429</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.429"></a>
-<span class="sourceLineNo">430</span> serializer.serialize(builder.build());<a name="line.430"></a>
-<span class="sourceLineNo">431</span> }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span> @Override<a name="line.433"></a>
-<span class="sourceLineNo">434</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.434"></a>
-<span class="sourceLineNo">435</span> throws IOException {<a name="line.435"></a>
-<span class="sourceLineNo">436</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.436"></a>
-<span class="sourceLineNo">437</span> ByteString dataString = bytesValue.getValue();<a name="line.437"></a>
-<span class="sourceLineNo">438</span><a name="line.438"></a>
-<span class="sourceLineNo">439</span> if (dataString.isEmpty()) {<a name="line.439"></a>
-<span class="sourceLineNo">440</span> data = null;<a name="line.440"></a>
-<span class="sourceLineNo">441</span> } else {<a name="line.441"></a>
-<span class="sourceLineNo">442</span> data = dataString.toByteArray();<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span> }<a name="line.444"></a>
+<span class="sourceLineNo">375</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.375"></a>
+<span class="sourceLineNo">376</span> }<a name="line.376"></a>
+<span class="sourceLineNo">377</span><a name="line.377"></a>
+<span class="sourceLineNo">378</span> @Override<a name="line.378"></a>
+<span class="sourceLineNo">379</span> protected boolean abort(TEnv env) { return false; }<a name="line.379"></a>
+<span class="sourceLineNo">380</span><a name="line.380"></a>
+<span class="sourceLineNo">381</span> @Override<a name="line.381"></a>
+<span class="sourceLineNo">382</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.382"></a>
+<span class="sourceLineNo">383</span> throws IOException {<a name="line.383"></a>
+<span class="sourceLineNo">384</span> }<a name="line.384"></a>
+<span class="sourceLineNo">385</span><a name="line.385"></a>
+<span class="sourceLineNo">386</span> @Override<a name="line.386"></a>
+<span class="sourceLineNo">387</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.387"></a>
+<span class="sourceLineNo">388</span> throws IOException {<a name="line.388"></a>
+<span class="sourceLineNo">389</span> }<a name="line.389"></a>
+<span class="sourceLineNo">390</span> }<a name="line.390"></a>
+<span class="sourceLineNo">391</span><a name="line.391"></a>
+<span class="sourceLineNo">392</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.392"></a>
+<span class="sourceLineNo">393</span> private byte[] data = null;<a name="line.393"></a>
+<span class="sourceLineNo">394</span><a name="line.394"></a>
+<span class="sourceLineNo">395</span> public TestProcedure() {}<a name="line.395"></a>
+<span class="sourceLineNo">396</span><a name="line.396"></a>
+<span class="sourceLineNo">397</span> public TestProcedure(long procId) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> this(procId, 0);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> }<a name="line.399"></a>
+<span class="sourceLineNo">400</span><a name="line.400"></a>
+<span class="sourceLineNo">401</span> public TestProcedure(long procId, long parentId) {<a name="line.401"></a>
+<span class="sourceLineNo">402</span> this(procId, parentId, null);<a name="line.402"></a>
+<span class="sourceLineNo">403</span> }<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> this(procId, parentId, parentId, data);<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> setData(data);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> setProcId(procId);<a name="line.411"></a>
+<span class="sourceLineNo">412</span> if (parentId > 0) {<a name="line.412"></a>
+<span class="sourceLineNo">413</span> setParentProcId(parentId);<a name="line.413"></a>
+<span class="sourceLineNo">414</span> }<a name="line.414"></a>
+<span class="sourceLineNo">415</span> if (rootId > 0 || parentId > 0) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> setRootProcId(rootId);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> public void addStackId(final int index) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> addStackIndex(index);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span><a name="line.423"></a>
+<span class="sourceLineNo">424</span> public void setSuccessState() {<a name="line.424"></a>
+<span class="sourceLineNo">425</span> setState(ProcedureState.SUCCESS);<a name="line.425"></a>
+<span class="sourceLineNo">426</span> }<a name="line.426"></a>
+<span class="sourceLineNo">427</span><a name="line.427"></a>
+<span class="sourceLineNo">428</span> public void setData(final byte[] data) {<a name="line.428"></a>
+<span class="sourceLineNo">429</span> this.data = data;<a name="line.429"></a>
+<span class="sourceLineNo">430</span> }<a name="line.430"></a>
+<span class="sourceLineNo">431</span><a name="line.431"></a>
+<span class="sourceLineNo">432</span> @Override<a name="line.432"></a>
+<span class="sourceLineNo">433</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.433"></a>
+<span class="sourceLineNo">434</span> throws IOException {<a name="line.434"></a>
+<span class="sourceLineNo">435</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.435"></a>
+<span class="sourceLineNo">436</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.436"></a>
+<span class="sourceLineNo">437</span> serializer.serialize(builder.build());<a name="line.437"></a>
+<span class="sourceLineNo">438</span> }<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span> @Override<a name="line.440"></a>
+<span class="sourceLineNo">441</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.441"></a>
+<span class="sourceLineNo">442</span> throws IOException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> ByteString dataString = bytesValue.getValue();<a name="line.444"></a>
<span class="sourceLineNo">445</span><a name="line.445"></a>
-<span class="sourceLineNo">446</span> // Mark acquire/release lock functions public for test uses.<a name="line.446"></a>
-<span class="sourceLineNo">447</span> @Override<a name="line.447"></a>
-<span class="sourceLineNo">448</span> public LockState acquireLock(Void env) {<a name="line.448"></a>
-<span class="sourceLineNo">449</span> return LockState.LOCK_ACQUIRED;<a name="line.449"></a>
-<span class="sourceLineNo">450</span> }<a name="line.450"></a>
-<span class="sourceLineNo">451</span><a name="line.451"></a>
-<span class="sourceLineNo">452</span> @Override<a name="line.452"></a>
-<span class="sourceLineNo">453</span> public void releaseLock(Void env) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> // no-op<a name="line.454"></a>
-<span class="sourceLineNo">455</span> }<a name="line.455"></a>
-<span class="sourceLineNo">456</span> }<a name="line.456"></a>
-<span class="sourceLineNo">457</span><a name="line.457"></a>
-<span class="sourceLineNo">458</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.458"></a>
-<span class="sourceLineNo">459</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.459"></a>
-<span class="sourceLineNo">460</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.460"></a>
-<span class="sourceLineNo">461</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span> private Set<Long> procIds;<a name="line.463"></a>
-<span class="sourceLineNo">464</span> private long maxProcId = 0;<a name="line.464"></a>
-<span class="sourceLineNo">465</span><a name="line.465"></a>
-<span class="sourceLineNo">466</span> public LoadCounter() {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> this(null);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> }<a name="line.468"></a>
+<span class="sourceLineNo">446</span> if (dataString.isEmpty()) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> data = null;<a name="line.447"></a>
+<span class="sourceLineNo">448</span> } else {<a name="line.448"></a>
+<span class="sourceLineNo">449</span> data = dataString.toByteArray();<a name="line.449"></a>
+<span class="sourceLineNo">450</span> }<a name="line.450"></a>
+<span class="sourceLineNo">451</span> }<a name="line.451"></a>
+<span class="sourceLineNo">452</span><a name="line.452"></a>
+<span class="sourceLineNo">453</span> // Mark acquire/release lock functions public for test uses.<a name="line.453"></a>
+<span class="sourceLineNo">454</span> @Override<a name="line.454"></a>
+<span class="sourceLineNo">455</span> public LockState acquireLock(Void env) {<a name="line.455"></a>
+<span class="sourceLineNo">456</span> return LockState.LOCK_ACQUIRED;<a name="line.456"></a>
+<span class="sourceLineNo">457</span> }<a name="line.457"></a>
+<span class="sourceLineNo">458</span><a name="line.458"></a>
+<span class="sourceLineNo">459</span> @Override<a name="line.459"></a>
+<span class="sourceLineNo">460</span> public void releaseLock(Void env) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // no-op<a name="line.461"></a>
+<span class="sourceLineNo">462</span> }<a name="line.462"></a>
+<span class="sourceLineNo">463</span> }<a name="line.463"></a>
+<span class="sourceLineNo">464</span><a name="line.464"></a>
+<span class="sourceLineNo">465</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.465"></a>
+<span class="sourceLineNo">466</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.466"></a>
+<span class="sourceLineNo">467</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.468"></a>
<span class="sourceLineNo">469</span><a name="line.469"></a>
-<span class="sourceLineNo">470</span> public LoadCounter(final Set<Long> procIds) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> this.procIds = procIds;<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span> public void reset() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> reset(null);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> }<a name="line.476"></a>
-<span class="sourceLineNo">477</span><a name="line.477"></a>
-<span class="sourceLineNo">478</span> public void reset(final Set<Long> procIds) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> corrupted.clear();<a name="line.479"></a>
-<span class="sourceLineNo">480</span> completed.clear();<a name="line.480"></a>
-<span class="sourceLineNo">481</span> runnable.clear();<a name="line.481"></a>
-<span class="sourceLineNo">482</span> this.procIds = procIds;<a name="line.482"></a>
-<span class="sourceLineNo">483</span> this.maxProcId = 0;<a name="line.483"></a>
-<span class="sourceLineNo">484</span> }<a name="line.484"></a>
-<span class="sourceLineNo">485</span><a name="line.485"></a>
-<span class="sourceLineNo">486</span> public long getMaxProcId() {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> return maxProcId;<a name="line.487"></a>
-<span class="sourceLineNo">488</span> }<a name="line.488"></a>
-<span class="sourceLineNo">489</span><a name="line.489"></a>
-<span class="sourceLineNo">490</span> public ArrayList<Procedure> getRunnables() {<a name="line.490"></a>
-<span class="sourceLineNo">491</span> return runnable;<a name="line.491"></a>
-<span class="sourceLineNo">492</span> }<a name="line.492"></a>
-<span class="sourceLineNo">493</span><a name="line.493"></a>
-<span class="sourceLineNo">494</span> public int getRunnableCount() {<a name="line.494"></a>
-<span class="sourceLineNo">495</span> return runnable.size();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> public ArrayList<Procedure> getCompleted() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> return completed;<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span> public int getCompletedCount() {<a name="line.502"></a>
-<span class="sourceLineNo">503</span> return completed.size();<a name="line.503"></a>
-<span class="sourceLineNo">504</span> }<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span> public int getLoadedCount() {<a name="line.506"></a>
-<span class="sourceLineNo">507</span> return runnable.size() + completed.size();<a name="line.507"></a>
-<span class="sourceLineNo">508</span> }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span> public ArrayList<Procedure> getCorrupted() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> return corrupted;<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span> public int getCorruptedCount() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span> return corrupted.size();<a name="line.515"></a>
-<span class="sourceLineNo">516</span> }<a name="line.516"></a>
-<span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span> public boolean isRunnable(final long procId) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span> for (Procedure proc: runnable) {<a name="line.519"></a>
-<span class="sourceLineNo">520</span> if (proc.getProcId() == procId) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> return true;<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return false;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @Override<a name="line.527"></a>
-<span class="sourceLineNo">528</span> public void setMaxProcId(long maxProcId) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> this.maxProcId = maxProcId;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> @Override<a name="line.532"></a>
-<span class="sourceLineNo">533</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.533"></a>
-<span class="sourceLineNo">534</span> while (procIter.hasNext()) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> long procId;<a name="line.535"></a>
-<span class="sourceLineNo">536</span> if (procIter.isNextFinished()) {<a name="line.536"></a>
-<span class="sourceLineNo">537</span> Procedure<?> proc = procIter.next();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> procId = proc.getProcId();<a name="line.538"></a>
-<span class="sourceLineNo">539</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.539"></a>
-<span class="sourceLineNo">540</span> completed.add(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> } else {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> Procedure proc = procIter.next();<a name="line.542"></a>
-<span class="sourceLineNo">543</span> procId = proc.getProcId();<a name="line.543"></a>
-<span class="sourceLineNo">544</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> runnable.add(proc);<a name="line.545"></a>
-<span class="sourceLineNo">546</span> }<a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (procIds != null) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span> }<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> @Override<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.554"></a>
-<span class="sourceLineNo">555</span> while (procIter.hasNext()) {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> Procedure proc = procIter.next();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> corrupted.add(proc);<a name="line.558"></a>
-<span class="sourceLineNo">559</span> }<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span>}<a name="line.562"></a>
+<span class="sourceLineNo">470</span> private Set<Long> procIds;<a name="line.470"></a>
+<span class="sourceLineNo">471</span> private long maxProcId = 0;<a name="line.471"></a>
+<span class="sourceLineNo">472</span><a name="line.472"></a>
+<span class="sourceLineNo">473</span> public LoadCounter() {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> this(null);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> }<a name="line.475"></a>
+<span class="sourceLineNo">476</span><a name="line.476"></a>
+<span class="sourceLineNo">477</span> public LoadCounter(final Set<Long> procIds) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> this.procIds = procIds;<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span> public void reset() {<a name="line.481"></a>
+<span class="sourceLineNo">482</span> reset(null);<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span><a name="line.484"></a>
+<span class="sourceLineNo">485</span> public void reset(final Set<Long> procIds) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> corrupted.clear();<a name="line.486"></a>
+<span class="sourceLineNo">487</span> completed.clear();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> runnable.clear();<a name="line.488"></a>
+<span class="sourceLineNo">489</span> this.procIds = procIds;<a name="line.489"></a>
+<span class="sourceLineNo">490</span> this.maxProcId = 0;<a name="line.490"></a>
+<span class="sourceLineNo">491</span> }<a name="line.491"></a>
+<span class="sourceLineNo">492</span><a name="line.492"></a>
+<span class="sourceLineNo">493</span> public long getMaxProcId() {<a name="line.493"></a>
+<span class="sourceLineNo">494</span> return maxProcId;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span><a name="line.496"></a>
+<span class="sourceLineNo">497</span> public ArrayList<Procedure> getRunnables() {<a name="line.497"></a>
+<span class="sourceLineNo">498</span> return runnable;<a name="line.498"></a>
+<span class="sourceLineNo">499</span> }<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span> public int getRunnableCount() {<a name="line.501"></a>
+<span class="sourceLineNo">502</span> return runnable.size();<a name="line.502"></a>
+<span class="sourceLineNo">503</span> }<a name="line.503"></a>
+<span class="sourceLineNo">504</span><a name="line.504"></a>
+<span class="sourceLineNo">505</span> public ArrayList<Procedure> getCompleted() {<a name="line.505"></a>
+<span class="sourceLineNo">506</span> return completed;<a name="line.506"></a>
+<span class="sourceLineNo">507</span> }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span> public int getCompletedCount() {<a name="line.509"></a>
+<span class="sourceLineNo">510</span> return completed.size();<a name="line.510"></a>
+<span class="sourceLineNo">511</span> }<a name="line.511"></a>
+<span class="sourceLineNo">512</span><a name="line.512"></a>
+<span class="sourceLineNo">513</span> public int getLoadedCount() {<a name="line.513"></a>
+<span class="sourceLineNo">514</span> return runnable.size() + completed.size();<a name="line.514"></a>
+<span class="sourceLineNo">515</span> }<a name="line.515"></a>
+<span class="sourceLineNo">516</span><a name="line.516"></a>
+<span class="sourceLineNo">517</span> public ArrayList<Procedure> getCorrupted() {<a name="line.517"></a>
+<span class="sourceLineNo">518</span> return corrupted;<a name="line.518"></a>
+<span class="sourceLineNo">519</span> }<a name="line.519"></a>
+<span class="sourceLineNo">520</span><a name="line.520"></a>
+<span class="sourceLineNo">521</span> public int getCorruptedCount() {<a name="line.521"></a>
+<span class="sourceLineNo">522</span> return corrupted.size();<a name="line.522"></a>
+<span class="sourceLineNo">523</span> }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span> public boolean isRunnable(final long procId) {<a name="line.525"></a>
+<span class="sourceLineNo">526</span> for (Procedure proc: runnable) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span> if (proc.getProcId() == procId) {<a name="line.527"></a>
+<span class="sourceLineNo">528</span> return true;<a name="line.528"></a>
+<span class="sourceLineNo">529</span> }<a name="line.529"></a>
+<span class="sourceLineNo">530</span> }<a name="line.530"></a>
+<span class="sourceLineNo">531</span> return false;<a name="line.531"></a>
+<span class="sourceLineNo">532</span> }<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span> @Override<a name="line.534"></a>
+<span class="sourceLineNo">535</span> public void setMaxProcId(long maxProcId) {<a name="line.535"></a>
+<span class="sourceLineNo">536</span> this.maxProcId = maxProcId;<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span><a name="line.538"></a>
+<span class="sourceLineNo">539</span> @Override<a name="line.539"></a>
+<span class="sourceLineNo">540</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.540"></a>
+<span class="sourceLineNo">541</span> while (procIter.hasNext()) {<a name="line.541"></a>
+<span class="sourceLineNo">542</span> long procId;<a name="line.542"></a>
+<span class="sourceLineNo">543</span> if (procIter.isNextFinished()) {<a name="line.543"></a>
+<span class="sourceLineNo">544</span> Procedure<?> proc = procIter.next();<a name="line.544"></a>
+<span class="sourceLineNo">545</span> procId = proc.getProcId();<a name="line.545"></a>
+<span class="sourceLineNo">546</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.546"></a>
+<span class="sourceLineNo">547</span> completed.add(proc);<a name="line.547"></a>
+<span class="sourceLineNo">548</span> } else {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> Procedure proc = procIter.next();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> procId = proc.getProcId();<a name="line.550"></a>
+<span class="sourceLineNo">551</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.551"></a>
+<span class="sourceLineNo">552</span> runnable.add(proc);<a name="line.552"></a>
+<span class="sourceLineNo">553</span> }<a name="line.553"></a>
+<span class="sourceLineNo">554</span> if (procIds != null) {<a name="line.554"></a>
+<span class="sourceLineNo">555</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.555"></a>
+<span class="sourceLineNo">556</span> }<a name="line.556"></a>
+<span class="sourceLineNo">557</span> }<a name="line.557"></a>
+<span class="sourceLineNo">558</span> }<a name="line.558"></a>
+<span class="sourceLineNo">559</span><a name="line.559"></a>
+<span class="sourceLineNo">560</span> @Override<a name="line.560"></a>
+<span class="sourceLineNo">561</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.561"></a>
+<span class="sourceLineNo">562</span> while (procIter.hasNext()) {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> Procedure proc = procIter.next();<a name="line.563"></a>
+<span class="sourceLineNo">564</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.564"></a>
+<span class="sourceLineNo">565</span> corrupted.add(proc);<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span> }<a name="line.567"></a>
+<span class="sourceLineNo">568</span> }<a name="line.568"></a>
+<span class="sourceLineNo">569</span>}<a name="line.569"></a>
[11/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</spa
<TRUNCATED>
[07/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/downloads.html
----------------------------------------------------------------------
diff --git a/downloads.html b/downloads.html
index 4477377..7fd1387 100644
--- a/downloads.html
+++ b/downloads.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Apache HBase Downloads</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -423,7 +423,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/export_control.html
----------------------------------------------------------------------
diff --git a/export_control.html b/export_control.html
index e72ecc6..21864ba 100644
--- a/export_control.html
+++ b/export_control.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Export Control
@@ -331,7 +331,7 @@ for more details.</p>
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index cc09f3b..8391016 100644
--- a/index.html
+++ b/index.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Apache HBaseâ„¢ Home</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -411,7 +411,7 @@ Apache HBase is an open-source, distributed, versioned, non-relational database
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/integration.html
----------------------------------------------------------------------
diff --git a/integration.html b/integration.html
index 1259628..55a7338 100644
--- a/integration.html
+++ b/integration.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – CI Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -291,7 +291,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/issue-tracking.html
----------------------------------------------------------------------
diff --git a/issue-tracking.html b/issue-tracking.html
index 69e9980..2e754f5 100644
--- a/issue-tracking.html
+++ b/issue-tracking.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Issue Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -288,7 +288,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/license.html
----------------------------------------------------------------------
diff --git a/license.html b/license.html
index bf72e02..8b58bb3 100644
--- a/license.html
+++ b/license.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Licenses</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -491,7 +491,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/mail-lists.html
----------------------------------------------------------------------
diff --git a/mail-lists.html b/mail-lists.html
index 438e9a1..19f02fc 100644
--- a/mail-lists.html
+++ b/mail-lists.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Mailing Lists</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -341,7 +341,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/metrics.html
----------------------------------------------------------------------
diff --git a/metrics.html b/metrics.html
index 6007dab..40e2e0a 100644
--- a/metrics.html
+++ b/metrics.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Apache HBase (TM) Metrics
@@ -459,7 +459,7 @@ export HBASE_REGIONSERVER_OPTS="$HBASE_JMX_OPTS -Dcom.sun.management.jmxrem
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/old_news.html
----------------------------------------------------------------------
diff --git a/old_news.html b/old_news.html
index 8d5e04d..cf4755d 100644
--- a/old_news.html
+++ b/old_news.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Old Apache HBase (TM) News
@@ -440,7 +440,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/plugin-management.html
----------------------------------------------------------------------
diff --git a/plugin-management.html b/plugin-management.html
index 8c87815..04773f3 100644
--- a/plugin-management.html
+++ b/plugin-management.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Plugin Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -440,7 +440,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/plugins.html
----------------------------------------------------------------------
diff --git a/plugins.html b/plugins.html
index 7936398..80134f9 100644
--- a/plugins.html
+++ b/plugins.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Plugins</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -375,7 +375,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/poweredbyhbase.html
----------------------------------------------------------------------
diff --git a/poweredbyhbase.html b/poweredbyhbase.html
index e628a88..cf598eb 100644
--- a/poweredbyhbase.html
+++ b/poweredbyhbase.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Powered By Apache HBase™</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -769,7 +769,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/project-info.html
----------------------------------------------------------------------
diff --git a/project-info.html b/project-info.html
index 2ee1de8..9fd86f3 100644
--- a/project-info.html
+++ b/project-info.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Information</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -335,7 +335,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/project-reports.html
----------------------------------------------------------------------
diff --git a/project-reports.html b/project-reports.html
index 4f3fb29..d093b22 100644
--- a/project-reports.html
+++ b/project-reports.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Generated Reports</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -305,7 +305,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/project-summary.html
----------------------------------------------------------------------
diff --git a/project-summary.html b/project-summary.html
index f4109b9..6ef4813 100644
--- a/project-summary.html
+++ b/project-summary.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Summary</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -331,7 +331,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/pseudo-distributed.html
----------------------------------------------------------------------
diff --git a/pseudo-distributed.html b/pseudo-distributed.html
index 1e16b85..a4c9404 100644
--- a/pseudo-distributed.html
+++ b/pseudo-distributed.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Running Apache HBase (TM) in pseudo-distributed mode
@@ -308,7 +308,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/replication.html
----------------------------------------------------------------------
diff --git a/replication.html b/replication.html
index 582491f..c543369 100644
--- a/replication.html
+++ b/replication.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Apache HBase (TM) Replication
@@ -303,7 +303,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/resources.html
----------------------------------------------------------------------
diff --git a/resources.html b/resources.html
index 934c107..a8e64f6 100644
--- a/resources.html
+++ b/resources.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Other Apache HBase (TM) Resources</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -331,7 +331,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/source-repository.html
----------------------------------------------------------------------
diff --git a/source-repository.html b/source-repository.html
index 7c61147..d2cf3d5 100644
--- a/source-repository.html
+++ b/source-repository.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Source Code Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -299,7 +299,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/sponsors.html
----------------------------------------------------------------------
diff --git a/sponsors.html b/sponsors.html
index 9b3d76b..26b5221 100644
--- a/sponsors.html
+++ b/sponsors.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Apache HBase™ Sponsors</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -333,7 +333,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/supportingprojects.html
----------------------------------------------------------------------
diff --git a/supportingprojects.html b/supportingprojects.html
index 72892e0..13d2f7d 100644
--- a/supportingprojects.html
+++ b/supportingprojects.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Supporting Projects</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -520,7 +520,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/team-list.html
----------------------------------------------------------------------
diff --git a/team-list.html b/team-list.html
index beec3e1..039982f 100644
--- a/team-list.html
+++ b/team-list.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Team</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -748,7 +748,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/index-all.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/index-all.html b/testdevapidocs/index-all.html
index d8b1d64..1bbc8ca 100644
--- a/testdevapidocs/index-all.html
+++ b/testdevapidocs/index-all.html
@@ -46382,6 +46382,11 @@
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildLoad--">testChildLoad()</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures</a></dt>
<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildLoadWithRestartAfterChildSuccess--">testChildLoadWithRestartAfterChildSuccess()</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures</a></dt>
+<dd>
+<div class="block">Test the state setting that happens after store to WAL; in particular the bit where we
+ set the parent runnable again after its children have all completed successfully.</div>
+</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildLoadWithSteppedRestart--">testChildLoadWithSteppedRestart()</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.html#testChildNormalRollbackStateCount--">testChildNormalRollbackStateCount()</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.html" title="class in org.apache.hadoop.hbase.procedure2">TestStateMachineProcedure</a></dt>
@@ -65255,6 +65260,10 @@
</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.html#toDebugString-org.apache.hadoop.hbase.metrics.BaseSource-">toDebugString(BaseSource)</a></span> - Method in class org.apache.hadoop.hbase.test.<a href="org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.html" title="class in org.apache.hadoop.hbase.test">MetricsAssertHelperImpl</a></dt>
<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#toggleKillAfterStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">toggleKillAfterStoreUpdate(ProcedureExecutor<TEnv>)</a></span> - Static method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureTestingUtility</a></dt>
+<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#toggleKillAfterStoreUpdate">toggleKillAfterStoreUpdate</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a></dt>
+<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#toggleKillBeforeStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">toggleKillBeforeStoreUpdate(ProcedureExecutor<TEnv>)</a></span> - Static method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureTestingUtility</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#toggleKillBeforeStoreUpdate">toggleKillBeforeStoreUpdate</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
index 4837d80..ec56445 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
@@ -158,8 +158,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/TagUsage.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">TagUsage</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.CacheOnWriteType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">TestCacheOnWrite.CacheOnWriteType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/TagUsage.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">TagUsage</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/package-tree.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/package-tree.html
index dcaab8c..72d6388 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/package-tree.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/package-tree.html
@@ -576,15 +576,15 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/PerformanceEvaluation.Counter.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">PerformanceEvaluation.Counter</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ClusterManager.ServiceType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ClusterManager.ServiceType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/RESTApiClusterManager.Service.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">RESTApiClusterManager.Service</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.ACTION.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">IntegrationTestDDLMasterFailover.ACTION</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ScanPerformanceEvaluation.ScanCounter.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ScanPerformanceEvaluation.ScanCounter</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/IntegrationTestRegionReplicaPerf.Stat.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">IntegrationTestRegionReplicaPerf.Stat</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ResourceChecker.Phase.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ResourceChecker.Phase</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ScanPerformanceEvaluation.ScanCounter.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ScanPerformanceEvaluation.ScanCounter</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HBaseClusterManager.CommandProvider.Operation.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HBaseClusterManager.CommandProvider.Operation</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/IntegrationTestDDLMasterFailover.ACTION.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">IntegrationTestDDLMasterFailover.ACTION</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/RESTApiClusterManager.RoleCommand.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">RESTApiClusterManager.RoleCommand</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ResourceChecker.Phase.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ResourceChecker.Phase</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/PerformanceEvaluation.Counter.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">PerformanceEvaluation.Counter</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ClusterManager.ServiceType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ClusterManager.ServiceType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
index 8422f7c..48d63fc 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
@@ -117,7 +117,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.458">ProcedureTestingUtility.LoadCounter</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.465">ProcedureTestingUtility.LoadCounter</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a>
implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoader</pre>
</li>
@@ -277,7 +277,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>corrupted</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.459">corrupted</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.466">corrupted</a></pre>
</li>
</ul>
<a name="completed">
@@ -286,7 +286,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>completed</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.460">completed</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.467">completed</a></pre>
</li>
</ul>
<a name="runnable">
@@ -295,7 +295,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>runnable</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.461">runnable</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.468">runnable</a></pre>
</li>
</ul>
<a name="procIds">
@@ -304,7 +304,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>procIds</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.463">procIds</a></pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.470">procIds</a></pre>
</li>
</ul>
<a name="maxProcId">
@@ -313,7 +313,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockListLast">
<li class="blockList">
<h4>maxProcId</h4>
-<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.464">maxProcId</a></pre>
+<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.471">maxProcId</a></pre>
</li>
</ul>
</li>
@@ -330,7 +330,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>LoadCounter</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.466">LoadCounter</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.473">LoadCounter</a>()</pre>
</li>
</ul>
<a name="LoadCounter-java.util.Set-">
@@ -339,7 +339,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockListLast">
<li class="blockList">
<h4>LoadCounter</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.470">LoadCounter</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> procIds)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.477">LoadCounter</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> procIds)</pre>
</li>
</ul>
</li>
@@ -356,7 +356,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>reset</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.474">reset</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.481">reset</a>()</pre>
</li>
</ul>
<a name="reset-java.util.Set-">
@@ -365,7 +365,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>reset</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.478">reset</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> procIds)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.485">reset</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> procIds)</pre>
</li>
</ul>
<a name="getMaxProcId--">
@@ -374,7 +374,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getMaxProcId</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.486">getMaxProcId</a>()</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.493">getMaxProcId</a>()</pre>
</li>
</ul>
<a name="getRunnables--">
@@ -383,7 +383,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getRunnables</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.490">getRunnables</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.497">getRunnables</a>()</pre>
</li>
</ul>
<a name="getRunnableCount--">
@@ -392,7 +392,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getRunnableCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.494">getRunnableCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.501">getRunnableCount</a>()</pre>
</li>
</ul>
<a name="getCompleted--">
@@ -401,7 +401,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getCompleted</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.498">getCompleted</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.505">getCompleted</a>()</pre>
</li>
</ul>
<a name="getCompletedCount--">
@@ -410,7 +410,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getCompletedCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.502">getCompletedCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.509">getCompletedCount</a>()</pre>
</li>
</ul>
<a name="getLoadedCount--">
@@ -419,7 +419,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getLoadedCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.506">getLoadedCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.513">getLoadedCount</a>()</pre>
</li>
</ul>
<a name="getCorrupted--">
@@ -428,7 +428,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getCorrupted</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.510">getCorrupted</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/ArrayList.html?is-external=true" title="class or interface in java.util">ArrayList</a><org.apache.hadoop.hbase.procedure2.Procedure> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.517">getCorrupted</a>()</pre>
</li>
</ul>
<a name="getCorruptedCount--">
@@ -437,7 +437,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>getCorruptedCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.514">getCorruptedCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.521">getCorruptedCount</a>()</pre>
</li>
</ul>
<a name="isRunnable-long-">
@@ -446,7 +446,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>isRunnable</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.518">isRunnable</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.525">isRunnable</a>(long procId)</pre>
</li>
</ul>
<a name="setMaxProcId-long-">
@@ -455,7 +455,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>setMaxProcId</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.528">setMaxProcId</a>(long maxProcId)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.535">setMaxProcId</a>(long maxProcId)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>setMaxProcId</code> in interface <code>org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoader</code></dd>
@@ -468,7 +468,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockList">
<li class="blockList">
<h4>load</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.533">load</a>(org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator procIter)
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.540">load</a>(org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator procIter)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -484,7 +484,7 @@ implements org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoad
<ul class="blockListLast">
<li class="blockList">
<h4>handleCorrupted</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.554">handleCorrupted</a>(org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator procIter)
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html#line.561">handleCorrupted</a>(org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator procIter)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
index 32882b2..d79f920 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html
@@ -126,7 +126,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.358">ProcedureTestingUtility.NoopProcedure</a><TEnv>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.365">ProcedureTestingUtility.NoopProcedure</a><TEnv>
extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
</li>
</ul>
@@ -248,7 +248,7 @@ extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
<ul class="blockListLast">
<li class="blockList">
<h4>NoopProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.359">NoopProcedure</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.366">NoopProcedure</a>()</pre>
</li>
</ul>
</li>
@@ -267,7 +267,7 @@ extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
<ul class="blockList">
<li class="blockList">
<h4>execute</h4>
-<pre>protected org.apache.hadoop.hbase.procedure2.Procedure[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.362">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a> env)
+<pre>protected org.apache.hadoop.hbase.procedure2.Procedure[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.369">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a> env)
throws org.apache.hadoop.hbase.procedure2.ProcedureYieldException,
org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a></pre>
@@ -289,7 +289,7 @@ extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
<ul class="blockList">
<li class="blockList">
<h4>rollback</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.368">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a> env)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.375">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a> env)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a></pre>
<dl>
@@ -309,7 +309,7 @@ extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
<ul class="blockList">
<li class="blockList">
<h4>abort</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.372">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a> env)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.379">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>abort</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="type parameter in ProcedureTestingUtility.NoopProcedure">TEnv</a>></code></dd>
@@ -322,7 +322,7 @@ extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
<ul class="blockList">
<li class="blockList">
<h4>serializeStateData</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.375">serializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.382">serializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
@@ -338,7 +338,7 @@ extends org.apache.hadoop.hbase.procedure2.Procedure<TEnv></pre>
<ul class="blockListLast">
<li class="blockList">
<h4>deserializeStateData</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.380">deserializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html#line.387">deserializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
index 1303f19..ad34222 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
@@ -131,7 +131,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.385">ProcedureTestingUtility.TestProcedure</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.392">ProcedureTestingUtility.TestProcedure</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.NoopProcedure.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureTestingUtility.NoopProcedure</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>></pre>
</li>
</ul>
@@ -297,7 +297,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockListLast">
<li class="blockList">
<h4>data</h4>
-<pre>private byte[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.386">data</a></pre>
+<pre>private byte[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.393">data</a></pre>
</li>
</ul>
</li>
@@ -314,7 +314,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>TestProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.388">TestProcedure</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.395">TestProcedure</a>()</pre>
</li>
</ul>
<a name="TestProcedure-long-">
@@ -323,7 +323,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>TestProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.390">TestProcedure</a>(long procId)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.397">TestProcedure</a>(long procId)</pre>
</li>
</ul>
<a name="TestProcedure-long-long-">
@@ -332,7 +332,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>TestProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.394">TestProcedure</a>(long procId,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.401">TestProcedure</a>(long procId,
long parentId)</pre>
</li>
</ul>
@@ -342,7 +342,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>TestProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.398">TestProcedure</a>(long procId,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.405">TestProcedure</a>(long procId,
long parentId,
byte[] data)</pre>
</li>
@@ -353,7 +353,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockListLast">
<li class="blockList">
<h4>TestProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.402">TestProcedure</a>(long procId,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.409">TestProcedure</a>(long procId,
long parentId,
long rootId,
byte[] data)</pre>
@@ -373,7 +373,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>addStackId</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.413">addStackId</a>(int index)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.420">addStackId</a>(int index)</pre>
</li>
</ul>
<a name="setSuccessState--">
@@ -382,7 +382,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>setSuccessState</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.417">setSuccessState</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.424">setSuccessState</a>()</pre>
</li>
</ul>
<a name="setData-byte:A-">
@@ -391,7 +391,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>setData</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.421">setData</a>(byte[] data)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.428">setData</a>(byte[] data)</pre>
</li>
</ul>
<a name="serializeStateData-org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer-">
@@ -400,7 +400,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>serializeStateData</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.426">serializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.433">serializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
@@ -416,7 +416,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>deserializeStateData</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.434">deserializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.441">deserializeStateData</a>(org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer serializer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
@@ -432,7 +432,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockList">
<li class="blockList">
<h4>acquireLock</h4>
-<pre>public org.apache.hadoop.hbase.procedure2.Procedure.LockState <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.448">acquireLock</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a> env)</pre>
+<pre>public org.apache.hadoop.hbase.procedure2.Procedure.LockState <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.455">acquireLock</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code>acquireLock</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>></code></dd>
@@ -445,7 +445,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTest
<ul class="blockListLast">
<li class="blockList">
<h4>releaseLock</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.453">releaseLock</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a> env)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html#line.460">releaseLock</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code>releaseLock</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Void.html?is-external=true" title="class or interface in java.lang">Void</a>></code></dd>
[05/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
index 45dd2e4..4310a16 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.LoadCounter.html
@@ -175,399 +175,406 @@
<span class="sourceLineNo">167</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.167"></a>
<span class="sourceLineNo">168</span> }<a name="line.168"></a>
<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.170"></a>
-<span class="sourceLineNo">171</span> boolean value) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.172"></a>
-<span class="sourceLineNo">173</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.173"></a>
+<span class="sourceLineNo">170</span> public static <TEnv> void toggleKillAfterStoreUpdate(ProcedureExecutor<TEnv> procExecutor) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span> createExecutorTesting(procExecutor);<a name="line.171"></a>
+<span class="sourceLineNo">172</span> procExecutor.testing.killAfterStoreUpdate = !procExecutor.testing.killAfterStoreUpdate;<a name="line.172"></a>
+<span class="sourceLineNo">173</span> LOG.warn("Set Kill after store update to: " + procExecutor.testing.killAfterStoreUpdate);<a name="line.173"></a>
<span class="sourceLineNo">174</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.174"></a>
<span class="sourceLineNo">175</span> }<a name="line.175"></a>
<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.177"></a>
-<span class="sourceLineNo">178</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> if (procExecutor.testing == null) return;<a name="line.179"></a>
-<span class="sourceLineNo">180</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.180"></a>
-<span class="sourceLineNo">181</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.182"></a>
-<span class="sourceLineNo">183</span> 1, procExecutor.getCorePoolSize());<a name="line.183"></a>
-<span class="sourceLineNo">184</span> }<a name="line.184"></a>
-<span class="sourceLineNo">185</span> }<a name="line.185"></a>
-<span class="sourceLineNo">186</span><a name="line.186"></a>
-<span class="sourceLineNo">187</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.187"></a>
-<span class="sourceLineNo">188</span> throws IOException {<a name="line.188"></a>
-<span class="sourceLineNo">189</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.189"></a>
-<span class="sourceLineNo">190</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.190"></a>
-<span class="sourceLineNo">191</span> procStore.start(1);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.192"></a>
-<span class="sourceLineNo">193</span> try {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.194"></a>
-<span class="sourceLineNo">195</span> } finally {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> procStore.stop(false);<a name="line.196"></a>
-<span class="sourceLineNo">197</span> procExecutor.stop();<a name="line.197"></a>
-<span class="sourceLineNo">198</span> }<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> }<a name="line.203"></a>
-<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.205"></a>
-<span class="sourceLineNo">206</span> final long nonceGroup, final long nonce) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.207"></a>
-<span class="sourceLineNo">208</span> waitProcedure(procExecutor, procId);<a name="line.208"></a>
-<span class="sourceLineNo">209</span> return procId;<a name="line.209"></a>
+<span class="sourceLineNo">177</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.177"></a>
+<span class="sourceLineNo">178</span> boolean value) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.179"></a>
+<span class="sourceLineNo">180</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.180"></a>
+<span class="sourceLineNo">181</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span><a name="line.183"></a>
+<span class="sourceLineNo">184</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.184"></a>
+<span class="sourceLineNo">185</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.185"></a>
+<span class="sourceLineNo">186</span> if (procExecutor.testing == null) return;<a name="line.186"></a>
+<span class="sourceLineNo">187</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.187"></a>
+<span class="sourceLineNo">188</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.189"></a>
+<span class="sourceLineNo">190</span> 1, procExecutor.getCorePoolSize());<a name="line.190"></a>
+<span class="sourceLineNo">191</span> }<a name="line.191"></a>
+<span class="sourceLineNo">192</span> }<a name="line.192"></a>
+<span class="sourceLineNo">193</span><a name="line.193"></a>
+<span class="sourceLineNo">194</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.194"></a>
+<span class="sourceLineNo">195</span> throws IOException {<a name="line.195"></a>
+<span class="sourceLineNo">196</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.196"></a>
+<span class="sourceLineNo">197</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.197"></a>
+<span class="sourceLineNo">198</span> procStore.start(1);<a name="line.198"></a>
+<span class="sourceLineNo">199</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.199"></a>
+<span class="sourceLineNo">200</span> try {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.201"></a>
+<span class="sourceLineNo">202</span> } finally {<a name="line.202"></a>
+<span class="sourceLineNo">203</span> procStore.stop(false);<a name="line.203"></a>
+<span class="sourceLineNo">204</span> procExecutor.stop();<a name="line.204"></a>
+<span class="sourceLineNo">205</span> }<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.208"></a>
+<span class="sourceLineNo">209</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.209"></a>
<span class="sourceLineNo">210</span> }<a name="line.210"></a>
<span class="sourceLineNo">211</span><a name="line.211"></a>
-<span class="sourceLineNo">212</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
+<span class="sourceLineNo">212</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
<span class="sourceLineNo">213</span> final long nonceGroup, final long nonce) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.214"></a>
-<span class="sourceLineNo">215</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.215"></a>
-<span class="sourceLineNo">216</span> assertFalse(procId >= 0);<a name="line.216"></a>
-<span class="sourceLineNo">217</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.217"></a>
-<span class="sourceLineNo">218</span> }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.220"></a>
-<span class="sourceLineNo">221</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> Threads.sleepWithoutInterrupt(250);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> }<a name="line.223"></a>
-<span class="sourceLineNo">224</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.224"></a>
+<span class="sourceLineNo">214</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.214"></a>
+<span class="sourceLineNo">215</span> waitProcedure(procExecutor, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return procId;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span><a name="line.218"></a>
+<span class="sourceLineNo">219</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.219"></a>
+<span class="sourceLineNo">220</span> final long nonceGroup, final long nonce) {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.221"></a>
+<span class="sourceLineNo">222</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.222"></a>
+<span class="sourceLineNo">223</span> assertFalse(procId >= 0);<a name="line.223"></a>
+<span class="sourceLineNo">224</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.224"></a>
<span class="sourceLineNo">225</span> }<a name="line.225"></a>
<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.228"></a>
+<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.227"></a>
+<span class="sourceLineNo">228</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.228"></a>
<span class="sourceLineNo">229</span> Threads.sleepWithoutInterrupt(250);<a name="line.229"></a>
<span class="sourceLineNo">230</span> }<a name="line.230"></a>
-<span class="sourceLineNo">231</span> }<a name="line.231"></a>
-<span class="sourceLineNo">232</span><a name="line.232"></a>
-<span class="sourceLineNo">233</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.233"></a>
-<span class="sourceLineNo">234</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> waitProcedure(procExecutor, procIds[i]);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span><a name="line.238"></a>
-<span class="sourceLineNo">239</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.239"></a>
-<span class="sourceLineNo">240</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.240"></a>
-<span class="sourceLineNo">241</span> waitProcedure(procExecutor, procId);<a name="line.241"></a>
-<span class="sourceLineNo">242</span> }<a name="line.242"></a>
-<span class="sourceLineNo">243</span> }<a name="line.243"></a>
-<span class="sourceLineNo">244</span><a name="line.244"></a>
-<span class="sourceLineNo">245</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.245"></a>
-<span class="sourceLineNo">246</span> int stableRuns = 0;<a name="line.246"></a>
-<span class="sourceLineNo">247</span> while (stableRuns < 10) {<a name="line.247"></a>
-<span class="sourceLineNo">248</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span> stableRuns = 0;<a name="line.249"></a>
-<span class="sourceLineNo">250</span> Threads.sleepWithoutInterrupt(100);<a name="line.250"></a>
-<span class="sourceLineNo">251</span> } else {<a name="line.251"></a>
-<span class="sourceLineNo">252</span> stableRuns++;<a name="line.252"></a>
-<span class="sourceLineNo">253</span> Threads.sleepWithoutInterrupt(25);<a name="line.253"></a>
-<span class="sourceLineNo">254</span> }<a name="line.254"></a>
-<span class="sourceLineNo">255</span> }<a name="line.255"></a>
-<span class="sourceLineNo">256</span> }<a name="line.256"></a>
-<span class="sourceLineNo">257</span><a name="line.257"></a>
-<span class="sourceLineNo">258</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.258"></a>
-<span class="sourceLineNo">259</span> long procId) {<a name="line.259"></a>
-<span class="sourceLineNo">260</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.260"></a>
-<span class="sourceLineNo">261</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span><a name="line.263"></a>
-<span class="sourceLineNo">264</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.264"></a>
-<span class="sourceLineNo">265</span> long procId) {<a name="line.265"></a>
-<span class="sourceLineNo">266</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.266"></a>
-<span class="sourceLineNo">267</span> assertTrue("expected procedure result", result != null);<a name="line.267"></a>
-<span class="sourceLineNo">268</span> assertProcNotFailed(result);<a name="line.268"></a>
+<span class="sourceLineNo">231</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.231"></a>
+<span class="sourceLineNo">232</span> }<a name="line.232"></a>
+<span class="sourceLineNo">233</span><a name="line.233"></a>
+<span class="sourceLineNo">234</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.234"></a>
+<span class="sourceLineNo">235</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.235"></a>
+<span class="sourceLineNo">236</span> Threads.sleepWithoutInterrupt(250);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> }<a name="line.237"></a>
+<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">239</span><a name="line.239"></a>
+<span class="sourceLineNo">240</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.241"></a>
+<span class="sourceLineNo">242</span> waitProcedure(procExecutor, procIds[i]);<a name="line.242"></a>
+<span class="sourceLineNo">243</span> }<a name="line.243"></a>
+<span class="sourceLineNo">244</span> }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.247"></a>
+<span class="sourceLineNo">248</span> waitProcedure(procExecutor, procId);<a name="line.248"></a>
+<span class="sourceLineNo">249</span> }<a name="line.249"></a>
+<span class="sourceLineNo">250</span> }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> int stableRuns = 0;<a name="line.253"></a>
+<span class="sourceLineNo">254</span> while (stableRuns < 10) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.255"></a>
+<span class="sourceLineNo">256</span> stableRuns = 0;<a name="line.256"></a>
+<span class="sourceLineNo">257</span> Threads.sleepWithoutInterrupt(100);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> } else {<a name="line.258"></a>
+<span class="sourceLineNo">259</span> stableRuns++;<a name="line.259"></a>
+<span class="sourceLineNo">260</span> Threads.sleepWithoutInterrupt(25);<a name="line.260"></a>
+<span class="sourceLineNo">261</span> }<a name="line.261"></a>
+<span class="sourceLineNo">262</span> }<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.265"></a>
+<span class="sourceLineNo">266</span> long procId) {<a name="line.266"></a>
+<span class="sourceLineNo">267</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.267"></a>
+<span class="sourceLineNo">268</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.268"></a>
<span class="sourceLineNo">269</span> }<a name="line.269"></a>
<span class="sourceLineNo">270</span><a name="line.270"></a>
-<span class="sourceLineNo">271</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.271"></a>
-<span class="sourceLineNo">272</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.272"></a>
-<span class="sourceLineNo">273</span> }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.275"></a>
-<span class="sourceLineNo">276</span> final long procId) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.277"></a>
-<span class="sourceLineNo">278</span> assertTrue("expected procedure result", result != null);<a name="line.278"></a>
-<span class="sourceLineNo">279</span> return assertProcFailed(result);<a name="line.279"></a>
+<span class="sourceLineNo">271</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.271"></a>
+<span class="sourceLineNo">272</span> long procId) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.273"></a>
+<span class="sourceLineNo">274</span> assertTrue("expected procedure result", result != null);<a name="line.274"></a>
+<span class="sourceLineNo">275</span> assertProcNotFailed(result);<a name="line.275"></a>
+<span class="sourceLineNo">276</span> }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.278"></a>
+<span class="sourceLineNo">279</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.279"></a>
<span class="sourceLineNo">280</span> }<a name="line.280"></a>
<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.282"></a>
-<span class="sourceLineNo">283</span> assertEquals(true, result.isFailed());<a name="line.283"></a>
-<span class="sourceLineNo">284</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.284"></a>
-<span class="sourceLineNo">285</span> return getExceptionCause(result);<a name="line.285"></a>
-<span class="sourceLineNo">286</span> }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.288"></a>
-<span class="sourceLineNo">289</span> Throwable cause = assertProcFailed(result);<a name="line.289"></a>
-<span class="sourceLineNo">290</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.290"></a>
-<span class="sourceLineNo">291</span> }<a name="line.291"></a>
-<span class="sourceLineNo">292</span><a name="line.292"></a>
-<span class="sourceLineNo">293</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> Throwable cause = assertProcFailed(result);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.298"></a>
-<span class="sourceLineNo">299</span> Throwable cause = assertProcFailed(result);<a name="line.299"></a>
-<span class="sourceLineNo">300</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.300"></a>
-<span class="sourceLineNo">301</span> cause instanceof IllegalArgumentIOException);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> }<a name="line.302"></a>
-<span class="sourceLineNo">303</span><a name="line.303"></a>
-<span class="sourceLineNo">304</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.304"></a>
-<span class="sourceLineNo">305</span> assert procInfo.isFailed();<a name="line.305"></a>
-<span class="sourceLineNo">306</span> Throwable cause = procInfo.getException().getCause();<a name="line.306"></a>
-<span class="sourceLineNo">307</span> return cause == null ? procInfo.getException() : cause;<a name="line.307"></a>
-<span class="sourceLineNo">308</span> }<a name="line.308"></a>
-<span class="sourceLineNo">309</span><a name="line.309"></a>
-<span class="sourceLineNo">310</span> /**<a name="line.310"></a>
-<span class="sourceLineNo">311</span> * Run through all procedure flow states TWICE while also restarting<a name="line.311"></a>
-<span class="sourceLineNo">312</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.312"></a>
-<span class="sourceLineNo">313</span> *<a name="line.313"></a>
-<span class="sourceLineNo">314</span> *<p>It does<a name="line.314"></a>
-<span class="sourceLineNo">315</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.315"></a>
-<span class="sourceLineNo">316</span> * <li>Restart executor/store<a name="line.316"></a>
-<span class="sourceLineNo">317</span> * <li>Execute step N - and then save to store<a name="line.317"></a>
-<span class="sourceLineNo">318</span> * </ol><a name="line.318"></a>
-<span class="sourceLineNo">319</span> *<a name="line.319"></a>
-<span class="sourceLineNo">320</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.320"></a>
-<span class="sourceLineNo">321</span> * idempotent.<a name="line.321"></a>
-<span class="sourceLineNo">322</span> */<a name="line.322"></a>
-<span class="sourceLineNo">323</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.323"></a>
-<span class="sourceLineNo">324</span> final long procId) throws Exception {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.325"></a>
-<span class="sourceLineNo">326</span> }<a name="line.326"></a>
-<span class="sourceLineNo">327</span><a name="line.327"></a>
-<span class="sourceLineNo">328</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.328"></a>
-<span class="sourceLineNo">329</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.329"></a>
-<span class="sourceLineNo">330</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.330"></a>
-<span class="sourceLineNo">331</span> }<a name="line.331"></a>
-<span class="sourceLineNo">332</span><a name="line.332"></a>
-<span class="sourceLineNo">333</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.333"></a>
-<span class="sourceLineNo">334</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.334"></a>
-<span class="sourceLineNo">335</span> throws Exception {<a name="line.335"></a>
-<span class="sourceLineNo">336</span> Procedure proc = procExec.getProcedure(procId);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> waitProcedure(procExec, procId);<a name="line.337"></a>
-<span class="sourceLineNo">338</span> assertEquals(false, procExec.isRunning());<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> proc = procExec.getProcedure(procId);<a name="line.340"></a>
-<span class="sourceLineNo">341</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> if (customRestart != null) {<a name="line.342"></a>
-<span class="sourceLineNo">343</span> customRestart.run();<a name="line.343"></a>
-<span class="sourceLineNo">344</span> } else {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> restart(procExec);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> }<a name="line.346"></a>
-<span class="sourceLineNo">347</span> waitProcedure(procExec, procId);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span><a name="line.349"></a>
-<span class="sourceLineNo">350</span> assertEquals(true, procExec.isRunning());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> if (expectFailure) {<a name="line.351"></a>
-<span class="sourceLineNo">352</span> assertProcFailed(procExec, procId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> } else {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> assertProcNotFailed(procExec, procId);<a name="line.354"></a>
+<span class="sourceLineNo">282</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.282"></a>
+<span class="sourceLineNo">283</span> final long procId) {<a name="line.283"></a>
+<span class="sourceLineNo">284</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.284"></a>
+<span class="sourceLineNo">285</span> assertTrue("expected procedure result", result != null);<a name="line.285"></a>
+<span class="sourceLineNo">286</span> return assertProcFailed(result);<a name="line.286"></a>
+<span class="sourceLineNo">287</span> }<a name="line.287"></a>
+<span class="sourceLineNo">288</span><a name="line.288"></a>
+<span class="sourceLineNo">289</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.289"></a>
+<span class="sourceLineNo">290</span> assertEquals(true, result.isFailed());<a name="line.290"></a>
+<span class="sourceLineNo">291</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.291"></a>
+<span class="sourceLineNo">292</span> return getExceptionCause(result);<a name="line.292"></a>
+<span class="sourceLineNo">293</span> }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span> Throwable cause = assertProcFailed(result);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.297"></a>
+<span class="sourceLineNo">298</span> }<a name="line.298"></a>
+<span class="sourceLineNo">299</span><a name="line.299"></a>
+<span class="sourceLineNo">300</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.300"></a>
+<span class="sourceLineNo">301</span> Throwable cause = assertProcFailed(result);<a name="line.301"></a>
+<span class="sourceLineNo">302</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.302"></a>
+<span class="sourceLineNo">303</span> }<a name="line.303"></a>
+<span class="sourceLineNo">304</span><a name="line.304"></a>
+<span class="sourceLineNo">305</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.305"></a>
+<span class="sourceLineNo">306</span> Throwable cause = assertProcFailed(result);<a name="line.306"></a>
+<span class="sourceLineNo">307</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.307"></a>
+<span class="sourceLineNo">308</span> cause instanceof IllegalArgumentIOException);<a name="line.308"></a>
+<span class="sourceLineNo">309</span> }<a name="line.309"></a>
+<span class="sourceLineNo">310</span><a name="line.310"></a>
+<span class="sourceLineNo">311</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span> assert procInfo.isFailed();<a name="line.312"></a>
+<span class="sourceLineNo">313</span> Throwable cause = procInfo.getException().getCause();<a name="line.313"></a>
+<span class="sourceLineNo">314</span> return cause == null ? procInfo.getException() : cause;<a name="line.314"></a>
+<span class="sourceLineNo">315</span> }<a name="line.315"></a>
+<span class="sourceLineNo">316</span><a name="line.316"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Run through all procedure flow states TWICE while also restarting<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.319"></a>
+<span class="sourceLineNo">320</span> *<a name="line.320"></a>
+<span class="sourceLineNo">321</span> *<p>It does<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Restart executor/store<a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Execute step N - and then save to store<a name="line.324"></a>
+<span class="sourceLineNo">325</span> * </ol><a name="line.325"></a>
+<span class="sourceLineNo">326</span> *<a name="line.326"></a>
+<span class="sourceLineNo">327</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * idempotent.<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.330"></a>
+<span class="sourceLineNo">331</span> final long procId) throws Exception {<a name="line.331"></a>
+<span class="sourceLineNo">332</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.332"></a>
+<span class="sourceLineNo">333</span> }<a name="line.333"></a>
+<span class="sourceLineNo">334</span><a name="line.334"></a>
+<span class="sourceLineNo">335</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.335"></a>
+<span class="sourceLineNo">336</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.337"></a>
+<span class="sourceLineNo">338</span> }<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.340"></a>
+<span class="sourceLineNo">341</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.341"></a>
+<span class="sourceLineNo">342</span> throws Exception {<a name="line.342"></a>
+<span class="sourceLineNo">343</span> Procedure proc = procExec.getProcedure(procId);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> waitProcedure(procExec, procId);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> assertEquals(false, procExec.isRunning());<a name="line.345"></a>
+<span class="sourceLineNo">346</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.346"></a>
+<span class="sourceLineNo">347</span> proc = procExec.getProcedure(procId);<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.348"></a>
+<span class="sourceLineNo">349</span> if (customRestart != null) {<a name="line.349"></a>
+<span class="sourceLineNo">350</span> customRestart.run();<a name="line.350"></a>
+<span class="sourceLineNo">351</span> } else {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> restart(procExec);<a name="line.352"></a>
+<span class="sourceLineNo">353</span> }<a name="line.353"></a>
+<span class="sourceLineNo">354</span> waitProcedure(procExec, procId);<a name="line.354"></a>
<span class="sourceLineNo">355</span> }<a name="line.355"></a>
-<span class="sourceLineNo">356</span> }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.358"></a>
-<span class="sourceLineNo">359</span> public NoopProcedure() {}<a name="line.359"></a>
-<span class="sourceLineNo">360</span><a name="line.360"></a>
-<span class="sourceLineNo">361</span> @Override<a name="line.361"></a>
-<span class="sourceLineNo">362</span> protected Procedure[] execute(TEnv env)<a name="line.362"></a>
-<span class="sourceLineNo">363</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> return null;<a name="line.364"></a>
-<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span> @Override<a name="line.367"></a>
-<span class="sourceLineNo">368</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> @Override<a name="line.371"></a>
-<span class="sourceLineNo">372</span> protected boolean abort(TEnv env) { return false; }<a name="line.372"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span> assertEquals(true, procExec.isRunning());<a name="line.357"></a>
+<span class="sourceLineNo">358</span> if (expectFailure) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> assertProcFailed(procExec, procId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> } else {<a name="line.360"></a>
+<span class="sourceLineNo">361</span> assertProcNotFailed(procExec, procId);<a name="line.361"></a>
+<span class="sourceLineNo">362</span> }<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.365"></a>
+<span class="sourceLineNo">366</span> public NoopProcedure() {}<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span> @Override<a name="line.368"></a>
+<span class="sourceLineNo">369</span> protected Procedure[] execute(TEnv env)<a name="line.369"></a>
+<span class="sourceLineNo">370</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.370"></a>
+<span class="sourceLineNo">371</span> return null;<a name="line.371"></a>
+<span class="sourceLineNo">372</span> }<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
<span class="sourceLineNo">374</span> @Override<a name="line.374"></a>
-<span class="sourceLineNo">375</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.375"></a>
-<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> @Override<a name="line.379"></a>
-<span class="sourceLineNo">380</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.380"></a>
-<span class="sourceLineNo">381</span> throws IOException {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.385"></a>
-<span class="sourceLineNo">386</span> private byte[] data = null;<a name="line.386"></a>
-<span class="sourceLineNo">387</span><a name="line.387"></a>
-<span class="sourceLineNo">388</span> public TestProcedure() {}<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> public TestProcedure(long procId) {<a name="line.390"></a>
-<span class="sourceLineNo">391</span> this(procId, 0);<a name="line.391"></a>
-<span class="sourceLineNo">392</span> }<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span> public TestProcedure(long procId, long parentId) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span> this(procId, parentId, null);<a name="line.395"></a>
-<span class="sourceLineNo">396</span> }<a name="line.396"></a>
-<span class="sourceLineNo">397</span><a name="line.397"></a>
-<span class="sourceLineNo">398</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> this(procId, parentId, parentId, data);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> }<a name="line.400"></a>
-<span class="sourceLineNo">401</span><a name="line.401"></a>
-<span class="sourceLineNo">402</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> setData(data);<a name="line.403"></a>
-<span class="sourceLineNo">404</span> setProcId(procId);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> if (parentId > 0) {<a name="line.405"></a>
-<span class="sourceLineNo">406</span> setParentProcId(parentId);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> }<a name="line.407"></a>
-<span class="sourceLineNo">408</span> if (rootId > 0 || parentId > 0) {<a name="line.408"></a>
-<span class="sourceLineNo">409</span> setRootProcId(rootId);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> }<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span><a name="line.412"></a>
-<span class="sourceLineNo">413</span> public void addStackId(final int index) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span> addStackIndex(index);<a name="line.414"></a>
-<span class="sourceLineNo">415</span> }<a name="line.415"></a>
-<span class="sourceLineNo">416</span><a name="line.416"></a>
-<span class="sourceLineNo">417</span> public void setSuccessState() {<a name="line.417"></a>
-<span class="sourceLineNo">418</span> setState(ProcedureState.SUCCESS);<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span><a name="line.420"></a>
-<span class="sourceLineNo">421</span> public void setData(final byte[] data) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> this.data = data;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span><a name="line.424"></a>
-<span class="sourceLineNo">425</span> @Override<a name="line.425"></a>
-<span class="sourceLineNo">426</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.426"></a>
-<span class="sourceLineNo">427</span> throws IOException {<a name="line.427"></a>
-<span class="sourceLineNo">428</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.428"></a>
-<span class="sourceLineNo">429</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.429"></a>
-<span class="sourceLineNo">430</span> serializer.serialize(builder.build());<a name="line.430"></a>
-<span class="sourceLineNo">431</span> }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span> @Override<a name="line.433"></a>
-<span class="sourceLineNo">434</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.434"></a>
-<span class="sourceLineNo">435</span> throws IOException {<a name="line.435"></a>
-<span class="sourceLineNo">436</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.436"></a>
-<span class="sourceLineNo">437</span> ByteString dataString = bytesValue.getValue();<a name="line.437"></a>
-<span class="sourceLineNo">438</span><a name="line.438"></a>
-<span class="sourceLineNo">439</span> if (dataString.isEmpty()) {<a name="line.439"></a>
-<span class="sourceLineNo">440</span> data = null;<a name="line.440"></a>
-<span class="sourceLineNo">441</span> } else {<a name="line.441"></a>
-<span class="sourceLineNo">442</span> data = dataString.toByteArray();<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span> }<a name="line.444"></a>
+<span class="sourceLineNo">375</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.375"></a>
+<span class="sourceLineNo">376</span> }<a name="line.376"></a>
+<span class="sourceLineNo">377</span><a name="line.377"></a>
+<span class="sourceLineNo">378</span> @Override<a name="line.378"></a>
+<span class="sourceLineNo">379</span> protected boolean abort(TEnv env) { return false; }<a name="line.379"></a>
+<span class="sourceLineNo">380</span><a name="line.380"></a>
+<span class="sourceLineNo">381</span> @Override<a name="line.381"></a>
+<span class="sourceLineNo">382</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.382"></a>
+<span class="sourceLineNo">383</span> throws IOException {<a name="line.383"></a>
+<span class="sourceLineNo">384</span> }<a name="line.384"></a>
+<span class="sourceLineNo">385</span><a name="line.385"></a>
+<span class="sourceLineNo">386</span> @Override<a name="line.386"></a>
+<span class="sourceLineNo">387</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.387"></a>
+<span class="sourceLineNo">388</span> throws IOException {<a name="line.388"></a>
+<span class="sourceLineNo">389</span> }<a name="line.389"></a>
+<span class="sourceLineNo">390</span> }<a name="line.390"></a>
+<span class="sourceLineNo">391</span><a name="line.391"></a>
+<span class="sourceLineNo">392</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.392"></a>
+<span class="sourceLineNo">393</span> private byte[] data = null;<a name="line.393"></a>
+<span class="sourceLineNo">394</span><a name="line.394"></a>
+<span class="sourceLineNo">395</span> public TestProcedure() {}<a name="line.395"></a>
+<span class="sourceLineNo">396</span><a name="line.396"></a>
+<span class="sourceLineNo">397</span> public TestProcedure(long procId) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> this(procId, 0);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> }<a name="line.399"></a>
+<span class="sourceLineNo">400</span><a name="line.400"></a>
+<span class="sourceLineNo">401</span> public TestProcedure(long procId, long parentId) {<a name="line.401"></a>
+<span class="sourceLineNo">402</span> this(procId, parentId, null);<a name="line.402"></a>
+<span class="sourceLineNo">403</span> }<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> this(procId, parentId, parentId, data);<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> setData(data);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> setProcId(procId);<a name="line.411"></a>
+<span class="sourceLineNo">412</span> if (parentId > 0) {<a name="line.412"></a>
+<span class="sourceLineNo">413</span> setParentProcId(parentId);<a name="line.413"></a>
+<span class="sourceLineNo">414</span> }<a name="line.414"></a>
+<span class="sourceLineNo">415</span> if (rootId > 0 || parentId > 0) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> setRootProcId(rootId);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> public void addStackId(final int index) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> addStackIndex(index);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span><a name="line.423"></a>
+<span class="sourceLineNo">424</span> public void setSuccessState() {<a name="line.424"></a>
+<span class="sourceLineNo">425</span> setState(ProcedureState.SUCCESS);<a name="line.425"></a>
+<span class="sourceLineNo">426</span> }<a name="line.426"></a>
+<span class="sourceLineNo">427</span><a name="line.427"></a>
+<span class="sourceLineNo">428</span> public void setData(final byte[] data) {<a name="line.428"></a>
+<span class="sourceLineNo">429</span> this.data = data;<a name="line.429"></a>
+<span class="sourceLineNo">430</span> }<a name="line.430"></a>
+<span class="sourceLineNo">431</span><a name="line.431"></a>
+<span class="sourceLineNo">432</span> @Override<a name="line.432"></a>
+<span class="sourceLineNo">433</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.433"></a>
+<span class="sourceLineNo">434</span> throws IOException {<a name="line.434"></a>
+<span class="sourceLineNo">435</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.435"></a>
+<span class="sourceLineNo">436</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.436"></a>
+<span class="sourceLineNo">437</span> serializer.serialize(builder.build());<a name="line.437"></a>
+<span class="sourceLineNo">438</span> }<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span> @Override<a name="line.440"></a>
+<span class="sourceLineNo">441</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.441"></a>
+<span class="sourceLineNo">442</span> throws IOException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> ByteString dataString = bytesValue.getValue();<a name="line.444"></a>
<span class="sourceLineNo">445</span><a name="line.445"></a>
-<span class="sourceLineNo">446</span> // Mark acquire/release lock functions public for test uses.<a name="line.446"></a>
-<span class="sourceLineNo">447</span> @Override<a name="line.447"></a>
-<span class="sourceLineNo">448</span> public LockState acquireLock(Void env) {<a name="line.448"></a>
-<span class="sourceLineNo">449</span> return LockState.LOCK_ACQUIRED;<a name="line.449"></a>
-<span class="sourceLineNo">450</span> }<a name="line.450"></a>
-<span class="sourceLineNo">451</span><a name="line.451"></a>
-<span class="sourceLineNo">452</span> @Override<a name="line.452"></a>
-<span class="sourceLineNo">453</span> public void releaseLock(Void env) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> // no-op<a name="line.454"></a>
-<span class="sourceLineNo">455</span> }<a name="line.455"></a>
-<span class="sourceLineNo">456</span> }<a name="line.456"></a>
-<span class="sourceLineNo">457</span><a name="line.457"></a>
-<span class="sourceLineNo">458</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.458"></a>
-<span class="sourceLineNo">459</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.459"></a>
-<span class="sourceLineNo">460</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.460"></a>
-<span class="sourceLineNo">461</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span> private Set<Long> procIds;<a name="line.463"></a>
-<span class="sourceLineNo">464</span> private long maxProcId = 0;<a name="line.464"></a>
-<span class="sourceLineNo">465</span><a name="line.465"></a>
-<span class="sourceLineNo">466</span> public LoadCounter() {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> this(null);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> }<a name="line.468"></a>
+<span class="sourceLineNo">446</span> if (dataString.isEmpty()) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> data = null;<a name="line.447"></a>
+<span class="sourceLineNo">448</span> } else {<a name="line.448"></a>
+<span class="sourceLineNo">449</span> data = dataString.toByteArray();<a name="line.449"></a>
+<span class="sourceLineNo">450</span> }<a name="line.450"></a>
+<span class="sourceLineNo">451</span> }<a name="line.451"></a>
+<span class="sourceLineNo">452</span><a name="line.452"></a>
+<span class="sourceLineNo">453</span> // Mark acquire/release lock functions public for test uses.<a name="line.453"></a>
+<span class="sourceLineNo">454</span> @Override<a name="line.454"></a>
+<span class="sourceLineNo">455</span> public LockState acquireLock(Void env) {<a name="line.455"></a>
+<span class="sourceLineNo">456</span> return LockState.LOCK_ACQUIRED;<a name="line.456"></a>
+<span class="sourceLineNo">457</span> }<a name="line.457"></a>
+<span class="sourceLineNo">458</span><a name="line.458"></a>
+<span class="sourceLineNo">459</span> @Override<a name="line.459"></a>
+<span class="sourceLineNo">460</span> public void releaseLock(Void env) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // no-op<a name="line.461"></a>
+<span class="sourceLineNo">462</span> }<a name="line.462"></a>
+<span class="sourceLineNo">463</span> }<a name="line.463"></a>
+<span class="sourceLineNo">464</span><a name="line.464"></a>
+<span class="sourceLineNo">465</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.465"></a>
+<span class="sourceLineNo">466</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.466"></a>
+<span class="sourceLineNo">467</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.468"></a>
<span class="sourceLineNo">469</span><a name="line.469"></a>
-<span class="sourceLineNo">470</span> public LoadCounter(final Set<Long> procIds) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> this.procIds = procIds;<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span> public void reset() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> reset(null);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> }<a name="line.476"></a>
-<span class="sourceLineNo">477</span><a name="line.477"></a>
-<span class="sourceLineNo">478</span> public void reset(final Set<Long> procIds) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> corrupted.clear();<a name="line.479"></a>
-<span class="sourceLineNo">480</span> completed.clear();<a name="line.480"></a>
-<span class="sourceLineNo">481</span> runnable.clear();<a name="line.481"></a>
-<span class="sourceLineNo">482</span> this.procIds = procIds;<a name="line.482"></a>
-<span class="sourceLineNo">483</span> this.maxProcId = 0;<a name="line.483"></a>
-<span class="sourceLineNo">484</span> }<a name="line.484"></a>
-<span class="sourceLineNo">485</span><a name="line.485"></a>
-<span class="sourceLineNo">486</span> public long getMaxProcId() {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> return maxProcId;<a name="line.487"></a>
-<span class="sourceLineNo">488</span> }<a name="line.488"></a>
-<span class="sourceLineNo">489</span><a name="line.489"></a>
-<span class="sourceLineNo">490</span> public ArrayList<Procedure> getRunnables() {<a name="line.490"></a>
-<span class="sourceLineNo">491</span> return runnable;<a name="line.491"></a>
-<span class="sourceLineNo">492</span> }<a name="line.492"></a>
-<span class="sourceLineNo">493</span><a name="line.493"></a>
-<span class="sourceLineNo">494</span> public int getRunnableCount() {<a name="line.494"></a>
-<span class="sourceLineNo">495</span> return runnable.size();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> public ArrayList<Procedure> getCompleted() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> return completed;<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span> public int getCompletedCount() {<a name="line.502"></a>
-<span class="sourceLineNo">503</span> return completed.size();<a name="line.503"></a>
-<span class="sourceLineNo">504</span> }<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span> public int getLoadedCount() {<a name="line.506"></a>
-<span class="sourceLineNo">507</span> return runnable.size() + completed.size();<a name="line.507"></a>
-<span class="sourceLineNo">508</span> }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span> public ArrayList<Procedure> getCorrupted() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> return corrupted;<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span> public int getCorruptedCount() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span> return corrupted.size();<a name="line.515"></a>
-<span class="sourceLineNo">516</span> }<a name="line.516"></a>
-<span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span> public boolean isRunnable(final long procId) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span> for (Procedure proc: runnable) {<a name="line.519"></a>
-<span class="sourceLineNo">520</span> if (proc.getProcId() == procId) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> return true;<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return false;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @Override<a name="line.527"></a>
-<span class="sourceLineNo">528</span> public void setMaxProcId(long maxProcId) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> this.maxProcId = maxProcId;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> @Override<a name="line.532"></a>
-<span class="sourceLineNo">533</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.533"></a>
-<span class="sourceLineNo">534</span> while (procIter.hasNext()) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> long procId;<a name="line.535"></a>
-<span class="sourceLineNo">536</span> if (procIter.isNextFinished()) {<a name="line.536"></a>
-<span class="sourceLineNo">537</span> Procedure<?> proc = procIter.next();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> procId = proc.getProcId();<a name="line.538"></a>
-<span class="sourceLineNo">539</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.539"></a>
-<span class="sourceLineNo">540</span> completed.add(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> } else {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> Procedure proc = procIter.next();<a name="line.542"></a>
-<span class="sourceLineNo">543</span> procId = proc.getProcId();<a name="line.543"></a>
-<span class="sourceLineNo">544</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> runnable.add(proc);<a name="line.545"></a>
-<span class="sourceLineNo">546</span> }<a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (procIds != null) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span> }<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> @Override<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.554"></a>
-<span class="sourceLineNo">555</span> while (procIter.hasNext()) {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> Procedure proc = procIter.next();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> corrupted.add(proc);<a name="line.558"></a>
-<span class="sourceLineNo">559</span> }<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span>}<a name="line.562"></a>
+<span class="sourceLineNo">470</span> private Set<Long> procIds;<a name="line.470"></a>
+<span class="sourceLineNo">471</span> private long maxProcId = 0;<a name="line.471"></a>
+<span class="sourceLineNo">472</span><a name="line.472"></a>
+<span class="sourceLineNo">473</span> public LoadCounter() {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> this(null);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> }<a name="line.475"></a>
+<span class="sourceLineNo">476</span><a name="line.476"></a>
+<span class="sourceLineNo">477</span> public LoadCounter(final Set<Long> procIds) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> this.procIds = procIds;<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span> public void reset() {<a name="line.481"></a>
+<span class="sourceLineNo">482</span> reset(null);<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span><a name="line.484"></a>
+<span class="sourceLineNo">485</span> public void reset(final Set<Long> procIds) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> corrupted.clear();<a name="line.486"></a>
+<span class="sourceLineNo">487</span> completed.clear();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> runnable.clear();<a name="line.488"></a>
+<span class="sourceLineNo">489</span> this.procIds = procIds;<a name="line.489"></a>
+<span class="sourceLineNo">490</span> this.maxProcId = 0;<a name="line.490"></a>
+<span class="sourceLineNo">491</span> }<a name="line.491"></a>
+<span class="sourceLineNo">492</span><a name="line.492"></a>
+<span class="sourceLineNo">493</span> public long getMaxProcId() {<a name="line.493"></a>
+<span class="sourceLineNo">494</span> return maxProcId;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span><a name="line.496"></a>
+<span class="sourceLineNo">497</span> public ArrayList<Procedure> getRunnables() {<a name="line.497"></a>
+<span class="sourceLineNo">498</span> return runnable;<a name="line.498"></a>
+<span class="sourceLineNo">499</span> }<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span> public int getRunnableCount() {<a name="line.501"></a>
+<span class="sourceLineNo">502</span> return runnable.size();<a name="line.502"></a>
+<span class="sourceLineNo">503</span> }<a name="line.503"></a>
+<span class="sourceLineNo">504</span><a name="line.504"></a>
+<span class="sourceLineNo">505</span> public ArrayList<Procedure> getCompleted() {<a name="line.505"></a>
+<span class="sourceLineNo">506</span> return completed;<a name="line.506"></a>
+<span class="sourceLineNo">507</span> }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span> public int getCompletedCount() {<a name="line.509"></a>
+<span class="sourceLineNo">510</span> return completed.size();<a name="line.510"></a>
+<span class="sourceLineNo">511</span> }<a name="line.511"></a>
+<span class="sourceLineNo">512</span><a name="line.512"></a>
+<span class="sourceLineNo">513</span> public int getLoadedCount() {<a name="line.513"></a>
+<span class="sourceLineNo">514</span> return runnable.size() + completed.size();<a name="line.514"></a>
+<span class="sourceLineNo">515</span> }<a name="line.515"></a>
+<span class="sourceLineNo">516</span><a name="line.516"></a>
+<span class="sourceLineNo">517</span> public ArrayList<Procedure> getCorrupted() {<a name="line.517"></a>
+<span class="sourceLineNo">518</span> return corrupted;<a name="line.518"></a>
+<span class="sourceLineNo">519</span> }<a name="line.519"></a>
+<span class="sourceLineNo">520</span><a name="line.520"></a>
+<span class="sourceLineNo">521</span> public int getCorruptedCount() {<a name="line.521"></a>
+<span class="sourceLineNo">522</span> return corrupted.size();<a name="line.522"></a>
+<span class="sourceLineNo">523</span> }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span> public boolean isRunnable(final long procId) {<a name="line.525"></a>
+<span class="sourceLineNo">526</span> for (Procedure proc: runnable) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span> if (proc.getProcId() == procId) {<a name="line.527"></a>
+<span class="sourceLineNo">528</span> return true;<a name="line.528"></a>
+<span class="sourceLineNo">529</span> }<a name="line.529"></a>
+<span class="sourceLineNo">530</span> }<a name="line.530"></a>
+<span class="sourceLineNo">531</span> return false;<a name="line.531"></a>
+<span class="sourceLineNo">532</span> }<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span> @Override<a name="line.534"></a>
+<span class="sourceLineNo">535</span> public void setMaxProcId(long maxProcId) {<a name="line.535"></a>
+<span class="sourceLineNo">536</span> this.maxProcId = maxProcId;<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span><a name="line.538"></a>
+<span class="sourceLineNo">539</span> @Override<a name="line.539"></a>
+<span class="sourceLineNo">540</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.540"></a>
+<span class="sourceLineNo">541</span> while (procIter.hasNext()) {<a name="line.541"></a>
+<span class="sourceLineNo">542</span> long procId;<a name="line.542"></a>
+<span class="sourceLineNo">543</span> if (procIter.isNextFinished()) {<a name="line.543"></a>
+<span class="sourceLineNo">544</span> Procedure<?> proc = procIter.next();<a name="line.544"></a>
+<span class="sourceLineNo">545</span> procId = proc.getProcId();<a name="line.545"></a>
+<span class="sourceLineNo">546</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.546"></a>
+<span class="sourceLineNo">547</span> completed.add(proc);<a name="line.547"></a>
+<span class="sourceLineNo">548</span> } else {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> Procedure proc = procIter.next();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> procId = proc.getProcId();<a name="line.550"></a>
+<span class="sourceLineNo">551</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.551"></a>
+<span class="sourceLineNo">552</span> runnable.add(proc);<a name="line.552"></a>
+<span class="sourceLineNo">553</span> }<a name="line.553"></a>
+<span class="sourceLineNo">554</span> if (procIds != null) {<a name="line.554"></a>
+<span class="sourceLineNo">555</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.555"></a>
+<span class="sourceLineNo">556</span> }<a name="line.556"></a>
+<span class="sourceLineNo">557</span> }<a name="line.557"></a>
+<span class="sourceLineNo">558</span> }<a name="line.558"></a>
+<span class="sourceLineNo">559</span><a name="line.559"></a>
+<span class="sourceLineNo">560</span> @Override<a name="line.560"></a>
+<span class="sourceLineNo">561</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.561"></a>
+<span class="sourceLineNo">562</span> while (procIter.hasNext()) {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> Procedure proc = procIter.next();<a name="line.563"></a>
+<span class="sourceLineNo">564</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.564"></a>
+<span class="sourceLineNo">565</span> corrupted.add(proc);<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span> }<a name="line.567"></a>
+<span class="sourceLineNo">568</span> }<a name="line.568"></a>
+<span class="sourceLineNo">569</span>}<a name="line.569"></a>
[09/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span class="sourceLineNo">600</span><a name="line.600"></a>
-<span clas
<TRUNCATED>
[15/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<s
<TRUNCATED>
[17/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html b/devapidocs/org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html
index c05bcb5..8795a11 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html
@@ -104,7 +104,9 @@
<tbody>
<tr class="altColor">
<td class="colFirst"><code>(package private) <a href="../../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></code></td>
-<td class="colLast"><span class="typeNameLabel">ProcedureExecutor.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing">testing</a></span></code> </td>
+<td class="colLast"><span class="typeNameLabel">ProcedureExecutor.</span><code><span class="memberNameLink"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing">testing</a></span></code>
+<div class="block"><a href="../../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing"><code>ProcedureExecutor.testing</code></a> is non-null when ProcedureExecutor is being tested.</div>
+</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/package-summary.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/package-summary.html b/devapidocs/org/apache/hadoop/hbase/procedure2/package-summary.html
index 9281f0a..f05d57e 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/package-summary.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/package-summary.html
@@ -216,7 +216,9 @@
</tr>
<tr class="altColor">
<td class="colFirst"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></td>
-<td class="colLast"> </td>
+<td class="colLast">
+<div class="block">Class with parameters describing how to fail/die when in testing-context.</div>
+</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><TEnvironment></td>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html b/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
index 8f3e25a..12c3557 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
@@ -216,10 +216,10 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/StateMachineProcedure.Flow.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">StateMachineProcedure.Flow</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockedResourceType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockedResourceType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">Procedure.LockState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/StateMachineProcedure.Flow.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">StateMachineProcedure.Flow</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">RootProcedureState.State</span></a></li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/package-use.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/package-use.html b/devapidocs/org/apache/hadoop/hbase/procedure2/package-use.html
index e9d9c64..e0bb05c 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/package-use.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/package-use.html
@@ -566,7 +566,9 @@
<td class="colOne"><a href="../../../../../org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.ProcedureExecutorListener.html#org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> </td>
</tr>
<tr class="rowColor">
-<td class="colOne"><a href="../../../../../org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html#org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a> </td>
+<td class="colOne"><a href="../../../../../org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.Testing.html#org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a>
+<div class="block">Class with parameters describing how to fail/die when in testing-context.</div>
+</td>
</tr>
<tr class="altColor">
<td class="colOne"><a href="../../../../../org/apache/hadoop/hbase/procedure2/class-use/ProcedureExecutor.WorkerThread.html#org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a> </td>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html b/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
index 7b63aa1..c4b118c 100644
--- a/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
@@ -231,9 +231,9 @@
<ul>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/OperationQuota.OperationType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">OperationQuota.OperationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottleType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottleType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">SpaceViolationPolicy</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/RpcThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">RpcThrottlingException.Type</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottlingException.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/RpcThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">RpcThrottlingException.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">SpaceViolationPolicy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaScope.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaScope</span></a></li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
index f45bb99..45475a1 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
@@ -708,20 +708,20 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">CompactingMemStore.IndexType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/SplitLogWorker.TaskExecutor.Status.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">SplitLogWorker.TaskExecutor.Status</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MetricsRegionServerSourceFactoryImpl.FactoryStorage</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.Type.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TimeRangeTracker.Type</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.StepDirection.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">DefaultHeapMemoryTuner.StepDirection</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScanType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/FlushType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">FlushType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChunkCreator.ChunkType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ChunkCreator.ChunkType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">HRegion.FlushResult.Result</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.Action.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MemStoreCompactionStrategy.Action</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.LimitScope.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.LimitScope</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/Region.Operation.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">Region.Operation</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.Type.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TimeRangeTracker.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">CompactingMemStore.IndexType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/BloomType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">BloomType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScanType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MetricsRegionServerSourceFactoryImpl.FactoryStorage</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChunkCreator.ChunkType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ChunkCreator.ChunkType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.NextState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.LimitScope.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.LimitScope</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.Action.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MemStoreCompactionStrategy.Action</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/Region.Operation.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">Region.Operation</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/SplitLogWorker.TaskExecutor.Status.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">SplitLogWorker.TaskExecutor.Status</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
index 3bd22b5..b377318 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
@@ -130,8 +130,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.MatchCode.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">ScanQueryMatcher.MatchCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/StripeCompactionScanQueryMatcher.DropDeletesInOutput.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">StripeCompactionScanQueryMatcher.DropDeletesInOutput</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.MatchCode.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">ScanQueryMatcher.MatchCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/DeleteTracker.DeleteResult.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">DeleteTracker.DeleteResult</span></a></li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
index 46651a5..f0c55c8 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
@@ -247,9 +247,9 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">ProtobufLogReader.WALHdrResult</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/CompressionContext.DictionaryIndex.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">CompressionContext.DictionaryIndex</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/RingBufferTruck.Type.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">RingBufferTruck.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/CompressionContext.DictionaryIndex.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">CompressionContext.DictionaryIndex</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">ProtobufLogReader.WALHdrResult</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/rest/model/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/rest/model/package-tree.html b/devapidocs/org/apache/hadoop/hbase/rest/model/package-tree.html
index 034077c..c20ff47 100644
--- a/devapidocs/org/apache/hadoop/hbase/rest/model/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/rest/model/package-tree.html
@@ -110,8 +110,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.rest.model.<a href="../../../../../../org/apache/hadoop/hbase/rest/model/ScannerModel.FilterModel.ByteArrayComparableModel.ComparatorType.html" title="enum in org.apache.hadoop.hbase.rest.model"><span class="typeNameLink">ScannerModel.FilterModel.ByteArrayComparableModel.ComparatorType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.rest.model.<a href="../../../../../../org/apache/hadoop/hbase/rest/model/ScannerModel.FilterModel.FilterType.html" title="enum in org.apache.hadoop.hbase.rest.model"><span class="typeNameLink">ScannerModel.FilterModel.FilterType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.rest.model.<a href="../../../../../../org/apache/hadoop/hbase/rest/model/ScannerModel.FilterModel.ByteArrayComparableModel.ComparatorType.html" title="enum in org.apache.hadoop.hbase.rest.model"><span class="typeNameLink">ScannerModel.FilterModel.ByteArrayComparableModel.ComparatorType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/security/package-tree.html b/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
index 6ef281d..dfa02b5 100644
--- a/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
@@ -191,9 +191,9 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslUtil.QualityOfProtection.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslUtil.QualityOfProtection</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslStatus.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslStatus</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/AuthMethod.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">AuthMethod</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslUtil.QualityOfProtection.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslUtil.QualityOfProtection</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html b/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
index 2923faf..81a90a9 100644
--- a/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
@@ -199,8 +199,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftServerRunner.ImplType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftServerRunner.ImplType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">MetricsThriftServerSourceFactoryImpl.FactoryStorage</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftServerRunner.ImplType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftServerRunner.ImplType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftMetrics.ThriftServerType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftMetrics.ThriftServerType</span></a></li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/util/package-tree.html b/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
index 4e90595..2e32f29 100644
--- a/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
@@ -515,14 +515,14 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PoolMap.PoolType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PoolMap.PoolType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/ChecksumType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">ChecksumType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.UnsafeComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.UnsafeComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a><T>)</li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PrettyPrinter.Unit.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PrettyPrinter.Unit</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/HBaseFsck.ErrorReporter.ERROR_CODE.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">HBaseFsck.ErrorReporter.ERROR_CODE</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PoolMap.PoolType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PoolMap.PoolType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/IdReadWriteLock.ReferenceType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">IdReadWriteLock.ReferenceType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Order.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Order</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/HBaseFsck.ErrorReporter.ERROR_CODE.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">HBaseFsck.ErrorReporter.ERROR_CODE</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PrettyPrinter.Unit.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PrettyPrinter.Unit</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.PureJavaComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.PureJavaComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a><T>)</li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/ChecksumType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">ChecksumType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html b/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
index bcb1724..c6caa9b 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
@@ -191,8 +191,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/RegionGroupingProvider.Strategies.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">RegionGroupingProvider.Strategies</span></a></li>
<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/WALFactory.Providers.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">WALFactory.Providers</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/RegionGroupingProvider.Strategies.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">RegionGroupingProvider.Strategies</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
index e7752d5..7757245 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
@@ -16,11 +16,11 @@
<span class="sourceLineNo">008</span>@InterfaceAudience.Private<a name="line.8"></a>
<span class="sourceLineNo">009</span>public class Version {<a name="line.9"></a>
<span class="sourceLineNo">010</span> public static final String version = "3.0.0-SNAPSHOT";<a name="line.10"></a>
-<span class="sourceLineNo">011</span> public static final String revision = "e705cf1447dd7e44f0cc0297746a0f14135e2f2f";<a name="line.11"></a>
+<span class="sourceLineNo">011</span> public static final String revision = "1114a1a65e78a00f643ef15b9a13bc710cb80845";<a name="line.11"></a>
<span class="sourceLineNo">012</span> public static final String user = "jenkins";<a name="line.12"></a>
-<span class="sourceLineNo">013</span> public static final String date = "Tue Aug 14 14:38:23 UTC 2018";<a name="line.13"></a>
+<span class="sourceLineNo">013</span> public static final String date = "Wed Aug 15 14:39:17 UTC 2018";<a name="line.13"></a>
<span class="sourceLineNo">014</span> public static final String url = "git://jenkins-websites1.apache.org/home/jenkins/jenkins-slave/workspace/hbase_generate_website/hbase";<a name="line.14"></a>
-<span class="sourceLineNo">015</span> public static final String srcChecksum = "0d84c58b6fa1860ba76507b3d036fb9b";<a name="line.15"></a>
+<span class="sourceLineNo">015</span> public static final String srcChecksum = "b03e26b326c47068ee3c2d38b4181ca8";<a name="line.15"></a>
<span class="sourceLineNo">016</span>}<a name="line.16"></a>
[06/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
index 2e748eb..d00888c 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
@@ -18,7 +18,7 @@
catch(err) {
}
//-->
-var methods = {"i0":9,"i1":9,"i2":9,"i3":9,"i4":9,"i5":9,"i6":9,"i7":9,"i8":9,"i9":9,"i10":9,"i11":9,"i12":9,"i13":9,"i14":9,"i15":9,"i16":9,"i17":9,"i18":9,"i19":9,"i20":9,"i21":9,"i22":9,"i23":9,"i24":9,"i25":9,"i26":9,"i27":9,"i28":9,"i29":9,"i30":9,"i31":9,"i32":9,"i33":9,"i34":9};
+var methods = {"i0":9,"i1":9,"i2":9,"i3":9,"i4":9,"i5":9,"i6":9,"i7":9,"i8":9,"i9":9,"i10":9,"i11":9,"i12":9,"i13":9,"i14":9,"i15":9,"i16":9,"i17":9,"i18":9,"i19":9,"i20":9,"i21":9,"i22":9,"i23":9,"i24":9,"i25":9,"i26":9,"i27":9,"i28":9,"i29":9,"i30":9,"i31":9,"i32":9,"i33":9,"i34":9,"i35":9};
var tabs = {65535:["t0","All Methods"],1:["t1","Static Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -350,27 +350,31 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr id="i29" class="rowColor">
<td class="colFirst"><code>static <TEnv> void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#toggleKillBeforeStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">toggleKillBeforeStoreUpdate</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#toggleKillAfterStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">toggleKillAfterStoreUpdate</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
</tr>
<tr id="i30" class="altColor">
<td class="colFirst"><code>static <TEnv> void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitAllProcedures-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">waitAllProcedures</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#toggleKillBeforeStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">toggleKillBeforeStoreUpdate</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
</tr>
<tr id="i31" class="rowColor">
<td class="colFirst"><code>static <TEnv> void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitNoProcedureRunning-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">waitNoProcedureRunning</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitAllProcedures-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">waitAllProcedures</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
</tr>
<tr id="i32" class="altColor">
<td class="colFirst"><code>static <TEnv> void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitNoProcedureRunning-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">waitNoProcedureRunning</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</code> </td>
+</tr>
+<tr id="i33" class="rowColor">
+<td class="colFirst"><code>static <TEnv> void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitProcedure-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-long-">waitProcedure</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long procId)</code> </td>
</tr>
-<tr id="i33" class="rowColor">
+<tr id="i34" class="altColor">
<td class="colFirst"><code>static <TEnv> void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitProcedure-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-org.apache.hadoop.hbase.procedure2.Procedure-">waitProcedure</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
org.apache.hadoop.hbase.procedure2.Procedure proc)</code> </td>
</tr>
-<tr id="i34" class="altColor">
+<tr id="i35" class="rowColor">
<td class="colFirst"><code>static <TEnv> void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#waitProcedures-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-long...-">waitProcedures</a></span>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long... procIds)</code> </td>
@@ -590,13 +594,22 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.163">toggleKillBeforeStoreUpdate</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
</li>
</ul>
+<a name="toggleKillAfterStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>toggleKillAfterStoreUpdate</h4>
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.170">toggleKillAfterStoreUpdate</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
+</li>
+</ul>
<a name="setKillAndToggleBeforeStoreUpdate-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-boolean-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>setKillAndToggleBeforeStoreUpdate</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.170">setKillAndToggleBeforeStoreUpdate</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.177">setKillAndToggleBeforeStoreUpdate</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
boolean value)</pre>
</li>
</ul>
@@ -606,7 +619,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertSingleExecutorForKillTests</h4>
-<pre>private static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.177">assertSingleExecutorForKillTests</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
+<pre>private static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.184">assertSingleExecutorForKillTests</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
</li>
</ul>
<a name="submitAndWait-org.apache.hadoop.conf.Configuration-java.lang.Object-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -617,7 +630,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitAndWait</h4>
-<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.187">submitAndWait</a>(org.apache.hadoop.conf.Configuration conf,
+<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.194">submitAndWait</a>(org.apache.hadoop.conf.Configuration conf,
TEnv env,
org.apache.hadoop.hbase.procedure2.Procedure<TEnv> proc)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
@@ -633,7 +646,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitAndWait</h4>
-<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.201">submitAndWait</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.208">submitAndWait</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
org.apache.hadoop.hbase.procedure2.Procedure proc)</pre>
</li>
</ul>
@@ -643,7 +656,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitAndWait</h4>
-<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.205">submitAndWait</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.212">submitAndWait</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
org.apache.hadoop.hbase.procedure2.Procedure proc,
long nonceGroup,
long nonce)</pre>
@@ -655,7 +668,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedure</h4>
-<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.212">submitProcedure</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.219">submitProcedure</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
org.apache.hadoop.hbase.procedure2.Procedure proc,
long nonceGroup,
long nonce)</pre>
@@ -667,7 +680,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>waitProcedure</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.220">waitProcedure</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.227">waitProcedure</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
org.apache.hadoop.hbase.procedure2.Procedure proc)</pre>
</li>
</ul>
@@ -677,7 +690,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>waitProcedure</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.227">waitProcedure</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.234">waitProcedure</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long procId)</pre>
</li>
</ul>
@@ -687,7 +700,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>waitProcedures</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.233">waitProcedures</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.240">waitProcedures</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long... procIds)</pre>
</li>
</ul>
@@ -697,7 +710,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>waitAllProcedures</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.239">waitAllProcedures</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.246">waitAllProcedures</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
</li>
</ul>
<a name="waitNoProcedureRunning-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-">
@@ -706,7 +719,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>waitNoProcedureRunning</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.245">waitNoProcedureRunning</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.252">waitNoProcedureRunning</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor)</pre>
</li>
</ul>
<a name="assertProcNotYetCompleted-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-long-">
@@ -715,7 +728,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertProcNotYetCompleted</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.258">assertProcNotYetCompleted</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.265">assertProcNotYetCompleted</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long procId)</pre>
</li>
</ul>
@@ -725,7 +738,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertProcNotFailed</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.264">assertProcNotFailed</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.271">assertProcNotFailed</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long procId)</pre>
</li>
</ul>
@@ -735,7 +748,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertProcNotFailed</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.271">assertProcNotFailed</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.278">assertProcNotFailed</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
</li>
</ul>
<a name="assertProcFailed-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-long-">
@@ -744,7 +757,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertProcFailed</h4>
-<pre>public static <TEnv> <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.275">assertProcFailed</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
+<pre>public static <TEnv> <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.282">assertProcFailed</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExecutor,
long procId)</pre>
</li>
</ul>
@@ -754,7 +767,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertProcFailed</h4>
-<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.282">assertProcFailed</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
+<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.289">assertProcFailed</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
</li>
</ul>
<a name="assertIsAbortException-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -763,7 +776,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertIsAbortException</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.288">assertIsAbortException</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.295">assertIsAbortException</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
</li>
</ul>
<a name="assertIsTimeoutException-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -772,7 +785,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertIsTimeoutException</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.293">assertIsTimeoutException</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.300">assertIsTimeoutException</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
</li>
</ul>
<a name="assertIsIllegalArgumentException-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -781,7 +794,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>assertIsIllegalArgumentException</h4>
-<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.298">assertIsIllegalArgumentException</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
+<pre>public static void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.305">assertIsIllegalArgumentException</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> result)</pre>
</li>
</ul>
<a name="getExceptionCause-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -790,7 +803,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getExceptionCause</h4>
-<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.304">getExceptionCause</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> procInfo)</pre>
+<pre>public static <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Throwable.html?is-external=true" title="class or interface in java.lang">Throwable</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.311">getExceptionCause</a>(org.apache.hadoop.hbase.procedure2.Procedure<?> procInfo)</pre>
</li>
</ul>
<a name="testRecoveryAndDoubleExecution-org.apache.hadoop.hbase.procedure2.ProcedureExecutor-long-">
@@ -799,7 +812,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testRecoveryAndDoubleExecution</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.323">testRecoveryAndDoubleExecution</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExec,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.330">testRecoveryAndDoubleExecution</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExec,
long procId)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<div class="block">Run through all procedure flow states TWICE while also restarting
@@ -825,7 +838,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testRecoveryAndDoubleExecution</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.328">testRecoveryAndDoubleExecution</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExec,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.335">testRecoveryAndDoubleExecution</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExec,
long procId,
boolean expectFailure)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
@@ -841,7 +854,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>testRecoveryAndDoubleExecution</h4>
-<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.333">testRecoveryAndDoubleExecution</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExec,
+<pre>public static <TEnv> void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html#line.340">testRecoveryAndDoubleExecution</a>(org.apache.hadoop.hbase.procedure2.ProcedureExecutor<TEnv> procExec,
long procId,
boolean expectFailure,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true" title="class or interface in java.lang">Runnable</a> customRestart)
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
index bbc75dd..6bce878 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html
@@ -127,7 +127,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.170">TestChildProcedures.TestChildProcedure</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.196">TestChildProcedures.TestChildProcedure</a>
extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></pre>
</li>
</ul>
@@ -248,7 +248,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockListLast">
<li class="blockList">
<h4>TestChildProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.171">TestChildProcedure</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.197">TestChildProcedure</a>()</pre>
</li>
</ul>
</li>
@@ -265,7 +265,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockList">
<li class="blockList">
<h4>execute</h4>
-<pre>public org.apache.hadoop.hbase.procedure2.Procedure[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.174">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
+<pre>public org.apache.hadoop.hbase.procedure2.Procedure[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.200">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>execute</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></code></dd>
@@ -278,7 +278,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockList">
<li class="blockList">
<h4>rollback</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.185">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.211">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>rollback</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></code></dd>
@@ -291,7 +291,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockListLast">
<li class="blockList">
<h4>abort</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.189">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestChildProcedure.html#line.215">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>abort</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></code></dd>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
index 1ea29a1..8a9360e 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html
@@ -107,7 +107,7 @@
</dl>
<hr>
<br>
-<pre>private static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.194">TestChildProcedures.TestProcEnv</a>
+<pre>private static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.220">TestChildProcedures.TestProcEnv</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
</li>
</ul>
@@ -129,10 +129,14 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr class="altColor">
<td class="colFirst"><code>boolean</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#toggleKillBeforeStoreUpdate">toggleKillBeforeStoreUpdate</a></span></code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#toggleKillAfterStoreUpdate">toggleKillAfterStoreUpdate</a></span></code> </td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>boolean</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#toggleKillBeforeStoreUpdate">toggleKillBeforeStoreUpdate</a></span></code> </td>
+</tr>
+<tr class="altColor">
+<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#triggerRollbackOnChild">triggerRollbackOnChild</a></span></code> </td>
</tr>
</table>
@@ -190,7 +194,16 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>toggleKillBeforeStoreUpdate</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.195">toggleKillBeforeStoreUpdate</a></pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.221">toggleKillBeforeStoreUpdate</a></pre>
+</li>
+</ul>
+<a name="toggleKillAfterStoreUpdate">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>toggleKillAfterStoreUpdate</h4>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.222">toggleKillAfterStoreUpdate</a></pre>
</li>
</ul>
<a name="triggerRollbackOnChild">
@@ -199,7 +212,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>triggerRollbackOnChild</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.196">triggerRollbackOnChild</a></pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.223">triggerRollbackOnChild</a></pre>
</li>
</ul>
</li>
@@ -216,7 +229,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>TestProcEnv</h4>
-<pre>private <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.194">TestProcEnv</a>()</pre>
+<pre>private <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html#line.220">TestProcEnv</a>()</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
index 083b5bc9..411bff9 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html
@@ -127,7 +127,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.149">TestChildProcedures.TestRootProcedure</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.172">TestChildProcedures.TestRootProcedure</a>
extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></pre>
</li>
</ul>
@@ -248,7 +248,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockListLast">
<li class="blockList">
<h4>TestRootProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.150">TestRootProcedure</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.173">TestRootProcedure</a>()</pre>
</li>
</ul>
</li>
@@ -265,7 +265,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockList">
<li class="blockList">
<h4>execute</h4>
-<pre>public org.apache.hadoop.hbase.procedure2.Procedure[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.153">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
+<pre>public org.apache.hadoop.hbase.procedure2.Procedure[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.176">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>execute</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></code></dd>
@@ -278,7 +278,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockList">
<li class="blockList">
<h4>rollback</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.161">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.187">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>rollback</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></code></dd>
@@ -291,7 +291,7 @@ extends org.apache.hadoop.hbase.procedure2.SequentialProcedure<<a href="../..
<ul class="blockListLast">
<li class="blockList">
<h4>abort</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.165">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestRootProcedure.html#line.191">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code>abort</code> in class <code>org.apache.hadoop.hbase.procedure2.Procedure<<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.TestProcEnv.html" title="class in org.apache.hadoop.hbase.procedure2">TestChildProcedures.TestProcEnv</a>></code></dd>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
index fe0e66e..6eccab6 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html
@@ -18,7 +18,7 @@
catch(err) {
}
//-->
-var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -246,14 +246,21 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code>void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildLoadWithSteppedRestart--">testChildLoadWithSteppedRestart</a></span>()</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildLoadWithRestartAfterChildSuccess--">testChildLoadWithRestartAfterChildSuccess</a></span>()</code>
+<div class="block">Test the state setting that happens after store to WAL; in particular the bit where we
+ set the parent runnable again after its children have all completed successfully.</div>
+</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code>void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildRollbackLoad--">testChildRollbackLoad</a></span>()</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildLoadWithSteppedRestart--">testChildLoadWithSteppedRestart</a></span>()</code> </td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildRollbackLoad--">testChildRollbackLoad</a></span>()</code> </td>
+</tr>
+<tr id="i7" class="rowColor">
+<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#testChildRollbackLoadWithSteppedRestart--">testChildRollbackLoadWithSteppedRestart</a></span>()</code> </td>
</tr>
</table>
@@ -453,13 +460,30 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</dl>
</li>
</ul>
+<a name="testChildLoadWithRestartAfterChildSuccess--">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>testChildLoadWithRestartAfterChildSuccess</h4>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.119">testChildLoadWithRestartAfterChildSuccess</a>()
+ throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
+<div class="block">Test the state setting that happens after store to WAL; in particular the bit where we
+ set the parent runnable again after its children have all completed successfully.
+ See HBASE-20978.</div>
+<dl>
+<dt><span class="throwsLabel">Throws:</span></dt>
+<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></code></dd>
+</dl>
+</li>
+</ul>
<a name="testChildRollbackLoad--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>testChildRollbackLoad</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.113">testChildRollbackLoad</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.136">testChildRollbackLoad</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -473,7 +497,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testChildRollbackLoadWithSteppedRestart</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.126">testChildRollbackLoadWithSteppedRestart</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.149">testChildRollbackLoadWithSteppedRestart</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Exception.html?is-external=true" title="class or interface in java.lang">Exception</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -487,7 +511,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>assertProcFailed</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.142">assertProcFailed</a>(long procId)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/TestChildProcedures.html#line.165">assertProcFailed</a>(long procId)</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
index 8dfb8db..7a3434e 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
@@ -206,9 +206,9 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestYieldProcedures.TestStateMachineProcedure.State.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">TestYieldProcedures.TestStateMachineProcedure.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.TestSMProcedureState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">TestStateMachineProcedure.TestSMProcedureState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.TestStateMachineProcedure.State.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">TestProcedureRecovery.TestStateMachineProcedure.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestStateMachineProcedure.TestSMProcedureState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">TestStateMachineProcedure.TestSMProcedureState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/TestYieldProcedures.TestStateMachineProcedure.State.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">TestYieldProcedures.TestStateMachineProcedure.State</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
index ea63e96..b280f02 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
@@ -668,11 +668,11 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.ActionType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestMultiLogThreshold.ActionType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestAtomicOperation.TestStep.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestAtomicOperation.TestStep</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.Metric.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestRegionServerReadRequestMetrics.Metric</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.Manipulation.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">DataBlockEncodingTool.Manipulation</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.ActionType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestMultiLogThreshold.ActionType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.CacheOnWriteType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestCacheOnWriteInSchema.CacheOnWriteType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestRegionServerReadRequestMetrics.Metric.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestRegionServerReadRequestMetrics.Metric</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TestAtomicOperation.TestStep.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TestAtomicOperation.TestStep</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/test/package-tree.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/test/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/test/package-tree.html
index e23761b..1499365 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/test/package-tree.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/test/package-tree.html
@@ -253,10 +253,10 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.test.<a href="../../../../../org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.Counters.html" title="enum in org.apache.hadoop.hbase.test"><span class="typeNameLink">IntegrationTestLoadAndVerify.Counters</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.test.<a href="../../../../../org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.Generator.Counts.html" title="enum in org.apache.hadoop.hbase.test"><span class="typeNameLink">IntegrationTestBigLinkedList.Generator.Counts</span></a></li>
<li type="circle">org.apache.hadoop.hbase.test.<a href="../../../../../org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.Verify.Counts.html" title="enum in org.apache.hadoop.hbase.test"><span class="typeNameLink">IntegrationTestBigLinkedList.Verify.Counts</span></a></li>
<li type="circle">org.apache.hadoop.hbase.test.<a href="../../../../../org/apache/hadoop/hbase/test/IntegrationTestWithCellVisibilityLoadAndVerify.Counters.html" title="enum in org.apache.hadoop.hbase.test"><span class="typeNameLink">IntegrationTestWithCellVisibilityLoadAndVerify.Counters</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.test.<a href="../../../../../org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.Generator.Counts.html" title="enum in org.apache.hadoop.hbase.test"><span class="typeNameLink">IntegrationTestBigLinkedList.Generator.Counts</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.test.<a href="../../../../../org/apache/hadoop/hbase/test/IntegrationTestLoadAndVerify.Counters.html" title="enum in org.apache.hadoop.hbase.test"><span class="typeNameLink">IntegrationTestLoadAndVerify.Counters</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/org/apache/hadoop/hbase/wal/package-tree.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/org/apache/hadoop/hbase/wal/package-tree.html b/testdevapidocs/org/apache/hadoop/hbase/wal/package-tree.html
index b21ae88..5282059 100644
--- a/testdevapidocs/org/apache/hadoop/hbase/wal/package-tree.html
+++ b/testdevapidocs/org/apache/hadoop/hbase/wal/package-tree.html
@@ -142,8 +142,8 @@
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/IOTestProvider.AllowedOperations.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">IOTestProvider.AllowedOperations</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">TestWALSplit.Corruptions</span></a></li>
<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/FaultyFSLog.FailureType.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">FaultyFSLog.FailureType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/TestWALSplit.Corruptions.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">TestWALSplit.Corruptions</span></a></li>
</ul>
</li>
</ul>
[03/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
index 45dd2e4..4310a16 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.TestProcedure.html
@@ -175,399 +175,406 @@
<span class="sourceLineNo">167</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.167"></a>
<span class="sourceLineNo">168</span> }<a name="line.168"></a>
<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.170"></a>
-<span class="sourceLineNo">171</span> boolean value) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.172"></a>
-<span class="sourceLineNo">173</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.173"></a>
+<span class="sourceLineNo">170</span> public static <TEnv> void toggleKillAfterStoreUpdate(ProcedureExecutor<TEnv> procExecutor) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span> createExecutorTesting(procExecutor);<a name="line.171"></a>
+<span class="sourceLineNo">172</span> procExecutor.testing.killAfterStoreUpdate = !procExecutor.testing.killAfterStoreUpdate;<a name="line.172"></a>
+<span class="sourceLineNo">173</span> LOG.warn("Set Kill after store update to: " + procExecutor.testing.killAfterStoreUpdate);<a name="line.173"></a>
<span class="sourceLineNo">174</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.174"></a>
<span class="sourceLineNo">175</span> }<a name="line.175"></a>
<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.177"></a>
-<span class="sourceLineNo">178</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> if (procExecutor.testing == null) return;<a name="line.179"></a>
-<span class="sourceLineNo">180</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.180"></a>
-<span class="sourceLineNo">181</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.182"></a>
-<span class="sourceLineNo">183</span> 1, procExecutor.getCorePoolSize());<a name="line.183"></a>
-<span class="sourceLineNo">184</span> }<a name="line.184"></a>
-<span class="sourceLineNo">185</span> }<a name="line.185"></a>
-<span class="sourceLineNo">186</span><a name="line.186"></a>
-<span class="sourceLineNo">187</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.187"></a>
-<span class="sourceLineNo">188</span> throws IOException {<a name="line.188"></a>
-<span class="sourceLineNo">189</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.189"></a>
-<span class="sourceLineNo">190</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.190"></a>
-<span class="sourceLineNo">191</span> procStore.start(1);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.192"></a>
-<span class="sourceLineNo">193</span> try {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.194"></a>
-<span class="sourceLineNo">195</span> } finally {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> procStore.stop(false);<a name="line.196"></a>
-<span class="sourceLineNo">197</span> procExecutor.stop();<a name="line.197"></a>
-<span class="sourceLineNo">198</span> }<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> }<a name="line.203"></a>
-<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.205"></a>
-<span class="sourceLineNo">206</span> final long nonceGroup, final long nonce) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.207"></a>
-<span class="sourceLineNo">208</span> waitProcedure(procExecutor, procId);<a name="line.208"></a>
-<span class="sourceLineNo">209</span> return procId;<a name="line.209"></a>
+<span class="sourceLineNo">177</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.177"></a>
+<span class="sourceLineNo">178</span> boolean value) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.179"></a>
+<span class="sourceLineNo">180</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.180"></a>
+<span class="sourceLineNo">181</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span><a name="line.183"></a>
+<span class="sourceLineNo">184</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.184"></a>
+<span class="sourceLineNo">185</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.185"></a>
+<span class="sourceLineNo">186</span> if (procExecutor.testing == null) return;<a name="line.186"></a>
+<span class="sourceLineNo">187</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.187"></a>
+<span class="sourceLineNo">188</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.189"></a>
+<span class="sourceLineNo">190</span> 1, procExecutor.getCorePoolSize());<a name="line.190"></a>
+<span class="sourceLineNo">191</span> }<a name="line.191"></a>
+<span class="sourceLineNo">192</span> }<a name="line.192"></a>
+<span class="sourceLineNo">193</span><a name="line.193"></a>
+<span class="sourceLineNo">194</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.194"></a>
+<span class="sourceLineNo">195</span> throws IOException {<a name="line.195"></a>
+<span class="sourceLineNo">196</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.196"></a>
+<span class="sourceLineNo">197</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.197"></a>
+<span class="sourceLineNo">198</span> procStore.start(1);<a name="line.198"></a>
+<span class="sourceLineNo">199</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.199"></a>
+<span class="sourceLineNo">200</span> try {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.201"></a>
+<span class="sourceLineNo">202</span> } finally {<a name="line.202"></a>
+<span class="sourceLineNo">203</span> procStore.stop(false);<a name="line.203"></a>
+<span class="sourceLineNo">204</span> procExecutor.stop();<a name="line.204"></a>
+<span class="sourceLineNo">205</span> }<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.208"></a>
+<span class="sourceLineNo">209</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.209"></a>
<span class="sourceLineNo">210</span> }<a name="line.210"></a>
<span class="sourceLineNo">211</span><a name="line.211"></a>
-<span class="sourceLineNo">212</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
+<span class="sourceLineNo">212</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
<span class="sourceLineNo">213</span> final long nonceGroup, final long nonce) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.214"></a>
-<span class="sourceLineNo">215</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.215"></a>
-<span class="sourceLineNo">216</span> assertFalse(procId >= 0);<a name="line.216"></a>
-<span class="sourceLineNo">217</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.217"></a>
-<span class="sourceLineNo">218</span> }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.220"></a>
-<span class="sourceLineNo">221</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> Threads.sleepWithoutInterrupt(250);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> }<a name="line.223"></a>
-<span class="sourceLineNo">224</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.224"></a>
+<span class="sourceLineNo">214</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.214"></a>
+<span class="sourceLineNo">215</span> waitProcedure(procExecutor, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return procId;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span><a name="line.218"></a>
+<span class="sourceLineNo">219</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.219"></a>
+<span class="sourceLineNo">220</span> final long nonceGroup, final long nonce) {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.221"></a>
+<span class="sourceLineNo">222</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.222"></a>
+<span class="sourceLineNo">223</span> assertFalse(procId >= 0);<a name="line.223"></a>
+<span class="sourceLineNo">224</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.224"></a>
<span class="sourceLineNo">225</span> }<a name="line.225"></a>
<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.228"></a>
+<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.227"></a>
+<span class="sourceLineNo">228</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.228"></a>
<span class="sourceLineNo">229</span> Threads.sleepWithoutInterrupt(250);<a name="line.229"></a>
<span class="sourceLineNo">230</span> }<a name="line.230"></a>
-<span class="sourceLineNo">231</span> }<a name="line.231"></a>
-<span class="sourceLineNo">232</span><a name="line.232"></a>
-<span class="sourceLineNo">233</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.233"></a>
-<span class="sourceLineNo">234</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> waitProcedure(procExecutor, procIds[i]);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span><a name="line.238"></a>
-<span class="sourceLineNo">239</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.239"></a>
-<span class="sourceLineNo">240</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.240"></a>
-<span class="sourceLineNo">241</span> waitProcedure(procExecutor, procId);<a name="line.241"></a>
-<span class="sourceLineNo">242</span> }<a name="line.242"></a>
-<span class="sourceLineNo">243</span> }<a name="line.243"></a>
-<span class="sourceLineNo">244</span><a name="line.244"></a>
-<span class="sourceLineNo">245</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.245"></a>
-<span class="sourceLineNo">246</span> int stableRuns = 0;<a name="line.246"></a>
-<span class="sourceLineNo">247</span> while (stableRuns < 10) {<a name="line.247"></a>
-<span class="sourceLineNo">248</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span> stableRuns = 0;<a name="line.249"></a>
-<span class="sourceLineNo">250</span> Threads.sleepWithoutInterrupt(100);<a name="line.250"></a>
-<span class="sourceLineNo">251</span> } else {<a name="line.251"></a>
-<span class="sourceLineNo">252</span> stableRuns++;<a name="line.252"></a>
-<span class="sourceLineNo">253</span> Threads.sleepWithoutInterrupt(25);<a name="line.253"></a>
-<span class="sourceLineNo">254</span> }<a name="line.254"></a>
-<span class="sourceLineNo">255</span> }<a name="line.255"></a>
-<span class="sourceLineNo">256</span> }<a name="line.256"></a>
-<span class="sourceLineNo">257</span><a name="line.257"></a>
-<span class="sourceLineNo">258</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.258"></a>
-<span class="sourceLineNo">259</span> long procId) {<a name="line.259"></a>
-<span class="sourceLineNo">260</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.260"></a>
-<span class="sourceLineNo">261</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span><a name="line.263"></a>
-<span class="sourceLineNo">264</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.264"></a>
-<span class="sourceLineNo">265</span> long procId) {<a name="line.265"></a>
-<span class="sourceLineNo">266</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.266"></a>
-<span class="sourceLineNo">267</span> assertTrue("expected procedure result", result != null);<a name="line.267"></a>
-<span class="sourceLineNo">268</span> assertProcNotFailed(result);<a name="line.268"></a>
+<span class="sourceLineNo">231</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.231"></a>
+<span class="sourceLineNo">232</span> }<a name="line.232"></a>
+<span class="sourceLineNo">233</span><a name="line.233"></a>
+<span class="sourceLineNo">234</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.234"></a>
+<span class="sourceLineNo">235</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.235"></a>
+<span class="sourceLineNo">236</span> Threads.sleepWithoutInterrupt(250);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> }<a name="line.237"></a>
+<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">239</span><a name="line.239"></a>
+<span class="sourceLineNo">240</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.241"></a>
+<span class="sourceLineNo">242</span> waitProcedure(procExecutor, procIds[i]);<a name="line.242"></a>
+<span class="sourceLineNo">243</span> }<a name="line.243"></a>
+<span class="sourceLineNo">244</span> }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.247"></a>
+<span class="sourceLineNo">248</span> waitProcedure(procExecutor, procId);<a name="line.248"></a>
+<span class="sourceLineNo">249</span> }<a name="line.249"></a>
+<span class="sourceLineNo">250</span> }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> int stableRuns = 0;<a name="line.253"></a>
+<span class="sourceLineNo">254</span> while (stableRuns < 10) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.255"></a>
+<span class="sourceLineNo">256</span> stableRuns = 0;<a name="line.256"></a>
+<span class="sourceLineNo">257</span> Threads.sleepWithoutInterrupt(100);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> } else {<a name="line.258"></a>
+<span class="sourceLineNo">259</span> stableRuns++;<a name="line.259"></a>
+<span class="sourceLineNo">260</span> Threads.sleepWithoutInterrupt(25);<a name="line.260"></a>
+<span class="sourceLineNo">261</span> }<a name="line.261"></a>
+<span class="sourceLineNo">262</span> }<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.265"></a>
+<span class="sourceLineNo">266</span> long procId) {<a name="line.266"></a>
+<span class="sourceLineNo">267</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.267"></a>
+<span class="sourceLineNo">268</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.268"></a>
<span class="sourceLineNo">269</span> }<a name="line.269"></a>
<span class="sourceLineNo">270</span><a name="line.270"></a>
-<span class="sourceLineNo">271</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.271"></a>
-<span class="sourceLineNo">272</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.272"></a>
-<span class="sourceLineNo">273</span> }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.275"></a>
-<span class="sourceLineNo">276</span> final long procId) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.277"></a>
-<span class="sourceLineNo">278</span> assertTrue("expected procedure result", result != null);<a name="line.278"></a>
-<span class="sourceLineNo">279</span> return assertProcFailed(result);<a name="line.279"></a>
+<span class="sourceLineNo">271</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.271"></a>
+<span class="sourceLineNo">272</span> long procId) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.273"></a>
+<span class="sourceLineNo">274</span> assertTrue("expected procedure result", result != null);<a name="line.274"></a>
+<span class="sourceLineNo">275</span> assertProcNotFailed(result);<a name="line.275"></a>
+<span class="sourceLineNo">276</span> }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.278"></a>
+<span class="sourceLineNo">279</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.279"></a>
<span class="sourceLineNo">280</span> }<a name="line.280"></a>
<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.282"></a>
-<span class="sourceLineNo">283</span> assertEquals(true, result.isFailed());<a name="line.283"></a>
-<span class="sourceLineNo">284</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.284"></a>
-<span class="sourceLineNo">285</span> return getExceptionCause(result);<a name="line.285"></a>
-<span class="sourceLineNo">286</span> }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.288"></a>
-<span class="sourceLineNo">289</span> Throwable cause = assertProcFailed(result);<a name="line.289"></a>
-<span class="sourceLineNo">290</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.290"></a>
-<span class="sourceLineNo">291</span> }<a name="line.291"></a>
-<span class="sourceLineNo">292</span><a name="line.292"></a>
-<span class="sourceLineNo">293</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> Throwable cause = assertProcFailed(result);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.298"></a>
-<span class="sourceLineNo">299</span> Throwable cause = assertProcFailed(result);<a name="line.299"></a>
-<span class="sourceLineNo">300</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.300"></a>
-<span class="sourceLineNo">301</span> cause instanceof IllegalArgumentIOException);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> }<a name="line.302"></a>
-<span class="sourceLineNo">303</span><a name="line.303"></a>
-<span class="sourceLineNo">304</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.304"></a>
-<span class="sourceLineNo">305</span> assert procInfo.isFailed();<a name="line.305"></a>
-<span class="sourceLineNo">306</span> Throwable cause = procInfo.getException().getCause();<a name="line.306"></a>
-<span class="sourceLineNo">307</span> return cause == null ? procInfo.getException() : cause;<a name="line.307"></a>
-<span class="sourceLineNo">308</span> }<a name="line.308"></a>
-<span class="sourceLineNo">309</span><a name="line.309"></a>
-<span class="sourceLineNo">310</span> /**<a name="line.310"></a>
-<span class="sourceLineNo">311</span> * Run through all procedure flow states TWICE while also restarting<a name="line.311"></a>
-<span class="sourceLineNo">312</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.312"></a>
-<span class="sourceLineNo">313</span> *<a name="line.313"></a>
-<span class="sourceLineNo">314</span> *<p>It does<a name="line.314"></a>
-<span class="sourceLineNo">315</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.315"></a>
-<span class="sourceLineNo">316</span> * <li>Restart executor/store<a name="line.316"></a>
-<span class="sourceLineNo">317</span> * <li>Execute step N - and then save to store<a name="line.317"></a>
-<span class="sourceLineNo">318</span> * </ol><a name="line.318"></a>
-<span class="sourceLineNo">319</span> *<a name="line.319"></a>
-<span class="sourceLineNo">320</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.320"></a>
-<span class="sourceLineNo">321</span> * idempotent.<a name="line.321"></a>
-<span class="sourceLineNo">322</span> */<a name="line.322"></a>
-<span class="sourceLineNo">323</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.323"></a>
-<span class="sourceLineNo">324</span> final long procId) throws Exception {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.325"></a>
-<span class="sourceLineNo">326</span> }<a name="line.326"></a>
-<span class="sourceLineNo">327</span><a name="line.327"></a>
-<span class="sourceLineNo">328</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.328"></a>
-<span class="sourceLineNo">329</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.329"></a>
-<span class="sourceLineNo">330</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.330"></a>
-<span class="sourceLineNo">331</span> }<a name="line.331"></a>
-<span class="sourceLineNo">332</span><a name="line.332"></a>
-<span class="sourceLineNo">333</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.333"></a>
-<span class="sourceLineNo">334</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.334"></a>
-<span class="sourceLineNo">335</span> throws Exception {<a name="line.335"></a>
-<span class="sourceLineNo">336</span> Procedure proc = procExec.getProcedure(procId);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> waitProcedure(procExec, procId);<a name="line.337"></a>
-<span class="sourceLineNo">338</span> assertEquals(false, procExec.isRunning());<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> proc = procExec.getProcedure(procId);<a name="line.340"></a>
-<span class="sourceLineNo">341</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> if (customRestart != null) {<a name="line.342"></a>
-<span class="sourceLineNo">343</span> customRestart.run();<a name="line.343"></a>
-<span class="sourceLineNo">344</span> } else {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> restart(procExec);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> }<a name="line.346"></a>
-<span class="sourceLineNo">347</span> waitProcedure(procExec, procId);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span><a name="line.349"></a>
-<span class="sourceLineNo">350</span> assertEquals(true, procExec.isRunning());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> if (expectFailure) {<a name="line.351"></a>
-<span class="sourceLineNo">352</span> assertProcFailed(procExec, procId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> } else {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> assertProcNotFailed(procExec, procId);<a name="line.354"></a>
+<span class="sourceLineNo">282</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.282"></a>
+<span class="sourceLineNo">283</span> final long procId) {<a name="line.283"></a>
+<span class="sourceLineNo">284</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.284"></a>
+<span class="sourceLineNo">285</span> assertTrue("expected procedure result", result != null);<a name="line.285"></a>
+<span class="sourceLineNo">286</span> return assertProcFailed(result);<a name="line.286"></a>
+<span class="sourceLineNo">287</span> }<a name="line.287"></a>
+<span class="sourceLineNo">288</span><a name="line.288"></a>
+<span class="sourceLineNo">289</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.289"></a>
+<span class="sourceLineNo">290</span> assertEquals(true, result.isFailed());<a name="line.290"></a>
+<span class="sourceLineNo">291</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.291"></a>
+<span class="sourceLineNo">292</span> return getExceptionCause(result);<a name="line.292"></a>
+<span class="sourceLineNo">293</span> }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span> Throwable cause = assertProcFailed(result);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.297"></a>
+<span class="sourceLineNo">298</span> }<a name="line.298"></a>
+<span class="sourceLineNo">299</span><a name="line.299"></a>
+<span class="sourceLineNo">300</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.300"></a>
+<span class="sourceLineNo">301</span> Throwable cause = assertProcFailed(result);<a name="line.301"></a>
+<span class="sourceLineNo">302</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.302"></a>
+<span class="sourceLineNo">303</span> }<a name="line.303"></a>
+<span class="sourceLineNo">304</span><a name="line.304"></a>
+<span class="sourceLineNo">305</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.305"></a>
+<span class="sourceLineNo">306</span> Throwable cause = assertProcFailed(result);<a name="line.306"></a>
+<span class="sourceLineNo">307</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.307"></a>
+<span class="sourceLineNo">308</span> cause instanceof IllegalArgumentIOException);<a name="line.308"></a>
+<span class="sourceLineNo">309</span> }<a name="line.309"></a>
+<span class="sourceLineNo">310</span><a name="line.310"></a>
+<span class="sourceLineNo">311</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span> assert procInfo.isFailed();<a name="line.312"></a>
+<span class="sourceLineNo">313</span> Throwable cause = procInfo.getException().getCause();<a name="line.313"></a>
+<span class="sourceLineNo">314</span> return cause == null ? procInfo.getException() : cause;<a name="line.314"></a>
+<span class="sourceLineNo">315</span> }<a name="line.315"></a>
+<span class="sourceLineNo">316</span><a name="line.316"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Run through all procedure flow states TWICE while also restarting<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.319"></a>
+<span class="sourceLineNo">320</span> *<a name="line.320"></a>
+<span class="sourceLineNo">321</span> *<p>It does<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Restart executor/store<a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Execute step N - and then save to store<a name="line.324"></a>
+<span class="sourceLineNo">325</span> * </ol><a name="line.325"></a>
+<span class="sourceLineNo">326</span> *<a name="line.326"></a>
+<span class="sourceLineNo">327</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * idempotent.<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.330"></a>
+<span class="sourceLineNo">331</span> final long procId) throws Exception {<a name="line.331"></a>
+<span class="sourceLineNo">332</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.332"></a>
+<span class="sourceLineNo">333</span> }<a name="line.333"></a>
+<span class="sourceLineNo">334</span><a name="line.334"></a>
+<span class="sourceLineNo">335</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.335"></a>
+<span class="sourceLineNo">336</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.337"></a>
+<span class="sourceLineNo">338</span> }<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.340"></a>
+<span class="sourceLineNo">341</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.341"></a>
+<span class="sourceLineNo">342</span> throws Exception {<a name="line.342"></a>
+<span class="sourceLineNo">343</span> Procedure proc = procExec.getProcedure(procId);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> waitProcedure(procExec, procId);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> assertEquals(false, procExec.isRunning());<a name="line.345"></a>
+<span class="sourceLineNo">346</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.346"></a>
+<span class="sourceLineNo">347</span> proc = procExec.getProcedure(procId);<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.348"></a>
+<span class="sourceLineNo">349</span> if (customRestart != null) {<a name="line.349"></a>
+<span class="sourceLineNo">350</span> customRestart.run();<a name="line.350"></a>
+<span class="sourceLineNo">351</span> } else {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> restart(procExec);<a name="line.352"></a>
+<span class="sourceLineNo">353</span> }<a name="line.353"></a>
+<span class="sourceLineNo">354</span> waitProcedure(procExec, procId);<a name="line.354"></a>
<span class="sourceLineNo">355</span> }<a name="line.355"></a>
-<span class="sourceLineNo">356</span> }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.358"></a>
-<span class="sourceLineNo">359</span> public NoopProcedure() {}<a name="line.359"></a>
-<span class="sourceLineNo">360</span><a name="line.360"></a>
-<span class="sourceLineNo">361</span> @Override<a name="line.361"></a>
-<span class="sourceLineNo">362</span> protected Procedure[] execute(TEnv env)<a name="line.362"></a>
-<span class="sourceLineNo">363</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> return null;<a name="line.364"></a>
-<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span> @Override<a name="line.367"></a>
-<span class="sourceLineNo">368</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> @Override<a name="line.371"></a>
-<span class="sourceLineNo">372</span> protected boolean abort(TEnv env) { return false; }<a name="line.372"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span> assertEquals(true, procExec.isRunning());<a name="line.357"></a>
+<span class="sourceLineNo">358</span> if (expectFailure) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> assertProcFailed(procExec, procId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> } else {<a name="line.360"></a>
+<span class="sourceLineNo">361</span> assertProcNotFailed(procExec, procId);<a name="line.361"></a>
+<span class="sourceLineNo">362</span> }<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.365"></a>
+<span class="sourceLineNo">366</span> public NoopProcedure() {}<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span> @Override<a name="line.368"></a>
+<span class="sourceLineNo">369</span> protected Procedure[] execute(TEnv env)<a name="line.369"></a>
+<span class="sourceLineNo">370</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.370"></a>
+<span class="sourceLineNo">371</span> return null;<a name="line.371"></a>
+<span class="sourceLineNo">372</span> }<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
<span class="sourceLineNo">374</span> @Override<a name="line.374"></a>
-<span class="sourceLineNo">375</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.375"></a>
-<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> @Override<a name="line.379"></a>
-<span class="sourceLineNo">380</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.380"></a>
-<span class="sourceLineNo">381</span> throws IOException {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.385"></a>
-<span class="sourceLineNo">386</span> private byte[] data = null;<a name="line.386"></a>
-<span class="sourceLineNo">387</span><a name="line.387"></a>
-<span class="sourceLineNo">388</span> public TestProcedure() {}<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> public TestProcedure(long procId) {<a name="line.390"></a>
-<span class="sourceLineNo">391</span> this(procId, 0);<a name="line.391"></a>
-<span class="sourceLineNo">392</span> }<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span> public TestProcedure(long procId, long parentId) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span> this(procId, parentId, null);<a name="line.395"></a>
-<span class="sourceLineNo">396</span> }<a name="line.396"></a>
-<span class="sourceLineNo">397</span><a name="line.397"></a>
-<span class="sourceLineNo">398</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> this(procId, parentId, parentId, data);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> }<a name="line.400"></a>
-<span class="sourceLineNo">401</span><a name="line.401"></a>
-<span class="sourceLineNo">402</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> setData(data);<a name="line.403"></a>
-<span class="sourceLineNo">404</span> setProcId(procId);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> if (parentId > 0) {<a name="line.405"></a>
-<span class="sourceLineNo">406</span> setParentProcId(parentId);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> }<a name="line.407"></a>
-<span class="sourceLineNo">408</span> if (rootId > 0 || parentId > 0) {<a name="line.408"></a>
-<span class="sourceLineNo">409</span> setRootProcId(rootId);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> }<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span><a name="line.412"></a>
-<span class="sourceLineNo">413</span> public void addStackId(final int index) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span> addStackIndex(index);<a name="line.414"></a>
-<span class="sourceLineNo">415</span> }<a name="line.415"></a>
-<span class="sourceLineNo">416</span><a name="line.416"></a>
-<span class="sourceLineNo">417</span> public void setSuccessState() {<a name="line.417"></a>
-<span class="sourceLineNo">418</span> setState(ProcedureState.SUCCESS);<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span><a name="line.420"></a>
-<span class="sourceLineNo">421</span> public void setData(final byte[] data) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> this.data = data;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span><a name="line.424"></a>
-<span class="sourceLineNo">425</span> @Override<a name="line.425"></a>
-<span class="sourceLineNo">426</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.426"></a>
-<span class="sourceLineNo">427</span> throws IOException {<a name="line.427"></a>
-<span class="sourceLineNo">428</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.428"></a>
-<span class="sourceLineNo">429</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.429"></a>
-<span class="sourceLineNo">430</span> serializer.serialize(builder.build());<a name="line.430"></a>
-<span class="sourceLineNo">431</span> }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span> @Override<a name="line.433"></a>
-<span class="sourceLineNo">434</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.434"></a>
-<span class="sourceLineNo">435</span> throws IOException {<a name="line.435"></a>
-<span class="sourceLineNo">436</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.436"></a>
-<span class="sourceLineNo">437</span> ByteString dataString = bytesValue.getValue();<a name="line.437"></a>
-<span class="sourceLineNo">438</span><a name="line.438"></a>
-<span class="sourceLineNo">439</span> if (dataString.isEmpty()) {<a name="line.439"></a>
-<span class="sourceLineNo">440</span> data = null;<a name="line.440"></a>
-<span class="sourceLineNo">441</span> } else {<a name="line.441"></a>
-<span class="sourceLineNo">442</span> data = dataString.toByteArray();<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span> }<a name="line.444"></a>
+<span class="sourceLineNo">375</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.375"></a>
+<span class="sourceLineNo">376</span> }<a name="line.376"></a>
+<span class="sourceLineNo">377</span><a name="line.377"></a>
+<span class="sourceLineNo">378</span> @Override<a name="line.378"></a>
+<span class="sourceLineNo">379</span> protected boolean abort(TEnv env) { return false; }<a name="line.379"></a>
+<span class="sourceLineNo">380</span><a name="line.380"></a>
+<span class="sourceLineNo">381</span> @Override<a name="line.381"></a>
+<span class="sourceLineNo">382</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.382"></a>
+<span class="sourceLineNo">383</span> throws IOException {<a name="line.383"></a>
+<span class="sourceLineNo">384</span> }<a name="line.384"></a>
+<span class="sourceLineNo">385</span><a name="line.385"></a>
+<span class="sourceLineNo">386</span> @Override<a name="line.386"></a>
+<span class="sourceLineNo">387</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.387"></a>
+<span class="sourceLineNo">388</span> throws IOException {<a name="line.388"></a>
+<span class="sourceLineNo">389</span> }<a name="line.389"></a>
+<span class="sourceLineNo">390</span> }<a name="line.390"></a>
+<span class="sourceLineNo">391</span><a name="line.391"></a>
+<span class="sourceLineNo">392</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.392"></a>
+<span class="sourceLineNo">393</span> private byte[] data = null;<a name="line.393"></a>
+<span class="sourceLineNo">394</span><a name="line.394"></a>
+<span class="sourceLineNo">395</span> public TestProcedure() {}<a name="line.395"></a>
+<span class="sourceLineNo">396</span><a name="line.396"></a>
+<span class="sourceLineNo">397</span> public TestProcedure(long procId) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> this(procId, 0);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> }<a name="line.399"></a>
+<span class="sourceLineNo">400</span><a name="line.400"></a>
+<span class="sourceLineNo">401</span> public TestProcedure(long procId, long parentId) {<a name="line.401"></a>
+<span class="sourceLineNo">402</span> this(procId, parentId, null);<a name="line.402"></a>
+<span class="sourceLineNo">403</span> }<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> this(procId, parentId, parentId, data);<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> setData(data);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> setProcId(procId);<a name="line.411"></a>
+<span class="sourceLineNo">412</span> if (parentId > 0) {<a name="line.412"></a>
+<span class="sourceLineNo">413</span> setParentProcId(parentId);<a name="line.413"></a>
+<span class="sourceLineNo">414</span> }<a name="line.414"></a>
+<span class="sourceLineNo">415</span> if (rootId > 0 || parentId > 0) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> setRootProcId(rootId);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> public void addStackId(final int index) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> addStackIndex(index);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span><a name="line.423"></a>
+<span class="sourceLineNo">424</span> public void setSuccessState() {<a name="line.424"></a>
+<span class="sourceLineNo">425</span> setState(ProcedureState.SUCCESS);<a name="line.425"></a>
+<span class="sourceLineNo">426</span> }<a name="line.426"></a>
+<span class="sourceLineNo">427</span><a name="line.427"></a>
+<span class="sourceLineNo">428</span> public void setData(final byte[] data) {<a name="line.428"></a>
+<span class="sourceLineNo">429</span> this.data = data;<a name="line.429"></a>
+<span class="sourceLineNo">430</span> }<a name="line.430"></a>
+<span class="sourceLineNo">431</span><a name="line.431"></a>
+<span class="sourceLineNo">432</span> @Override<a name="line.432"></a>
+<span class="sourceLineNo">433</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.433"></a>
+<span class="sourceLineNo">434</span> throws IOException {<a name="line.434"></a>
+<span class="sourceLineNo">435</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.435"></a>
+<span class="sourceLineNo">436</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.436"></a>
+<span class="sourceLineNo">437</span> serializer.serialize(builder.build());<a name="line.437"></a>
+<span class="sourceLineNo">438</span> }<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span> @Override<a name="line.440"></a>
+<span class="sourceLineNo">441</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.441"></a>
+<span class="sourceLineNo">442</span> throws IOException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> ByteString dataString = bytesValue.getValue();<a name="line.444"></a>
<span class="sourceLineNo">445</span><a name="line.445"></a>
-<span class="sourceLineNo">446</span> // Mark acquire/release lock functions public for test uses.<a name="line.446"></a>
-<span class="sourceLineNo">447</span> @Override<a name="line.447"></a>
-<span class="sourceLineNo">448</span> public LockState acquireLock(Void env) {<a name="line.448"></a>
-<span class="sourceLineNo">449</span> return LockState.LOCK_ACQUIRED;<a name="line.449"></a>
-<span class="sourceLineNo">450</span> }<a name="line.450"></a>
-<span class="sourceLineNo">451</span><a name="line.451"></a>
-<span class="sourceLineNo">452</span> @Override<a name="line.452"></a>
-<span class="sourceLineNo">453</span> public void releaseLock(Void env) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> // no-op<a name="line.454"></a>
-<span class="sourceLineNo">455</span> }<a name="line.455"></a>
-<span class="sourceLineNo">456</span> }<a name="line.456"></a>
-<span class="sourceLineNo">457</span><a name="line.457"></a>
-<span class="sourceLineNo">458</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.458"></a>
-<span class="sourceLineNo">459</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.459"></a>
-<span class="sourceLineNo">460</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.460"></a>
-<span class="sourceLineNo">461</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span> private Set<Long> procIds;<a name="line.463"></a>
-<span class="sourceLineNo">464</span> private long maxProcId = 0;<a name="line.464"></a>
-<span class="sourceLineNo">465</span><a name="line.465"></a>
-<span class="sourceLineNo">466</span> public LoadCounter() {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> this(null);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> }<a name="line.468"></a>
+<span class="sourceLineNo">446</span> if (dataString.isEmpty()) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> data = null;<a name="line.447"></a>
+<span class="sourceLineNo">448</span> } else {<a name="line.448"></a>
+<span class="sourceLineNo">449</span> data = dataString.toByteArray();<a name="line.449"></a>
+<span class="sourceLineNo">450</span> }<a name="line.450"></a>
+<span class="sourceLineNo">451</span> }<a name="line.451"></a>
+<span class="sourceLineNo">452</span><a name="line.452"></a>
+<span class="sourceLineNo">453</span> // Mark acquire/release lock functions public for test uses.<a name="line.453"></a>
+<span class="sourceLineNo">454</span> @Override<a name="line.454"></a>
+<span class="sourceLineNo">455</span> public LockState acquireLock(Void env) {<a name="line.455"></a>
+<span class="sourceLineNo">456</span> return LockState.LOCK_ACQUIRED;<a name="line.456"></a>
+<span class="sourceLineNo">457</span> }<a name="line.457"></a>
+<span class="sourceLineNo">458</span><a name="line.458"></a>
+<span class="sourceLineNo">459</span> @Override<a name="line.459"></a>
+<span class="sourceLineNo">460</span> public void releaseLock(Void env) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // no-op<a name="line.461"></a>
+<span class="sourceLineNo">462</span> }<a name="line.462"></a>
+<span class="sourceLineNo">463</span> }<a name="line.463"></a>
+<span class="sourceLineNo">464</span><a name="line.464"></a>
+<span class="sourceLineNo">465</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.465"></a>
+<span class="sourceLineNo">466</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.466"></a>
+<span class="sourceLineNo">467</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.468"></a>
<span class="sourceLineNo">469</span><a name="line.469"></a>
-<span class="sourceLineNo">470</span> public LoadCounter(final Set<Long> procIds) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> this.procIds = procIds;<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span> public void reset() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> reset(null);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> }<a name="line.476"></a>
-<span class="sourceLineNo">477</span><a name="line.477"></a>
-<span class="sourceLineNo">478</span> public void reset(final Set<Long> procIds) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> corrupted.clear();<a name="line.479"></a>
-<span class="sourceLineNo">480</span> completed.clear();<a name="line.480"></a>
-<span class="sourceLineNo">481</span> runnable.clear();<a name="line.481"></a>
-<span class="sourceLineNo">482</span> this.procIds = procIds;<a name="line.482"></a>
-<span class="sourceLineNo">483</span> this.maxProcId = 0;<a name="line.483"></a>
-<span class="sourceLineNo">484</span> }<a name="line.484"></a>
-<span class="sourceLineNo">485</span><a name="line.485"></a>
-<span class="sourceLineNo">486</span> public long getMaxProcId() {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> return maxProcId;<a name="line.487"></a>
-<span class="sourceLineNo">488</span> }<a name="line.488"></a>
-<span class="sourceLineNo">489</span><a name="line.489"></a>
-<span class="sourceLineNo">490</span> public ArrayList<Procedure> getRunnables() {<a name="line.490"></a>
-<span class="sourceLineNo">491</span> return runnable;<a name="line.491"></a>
-<span class="sourceLineNo">492</span> }<a name="line.492"></a>
-<span class="sourceLineNo">493</span><a name="line.493"></a>
-<span class="sourceLineNo">494</span> public int getRunnableCount() {<a name="line.494"></a>
-<span class="sourceLineNo">495</span> return runnable.size();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> public ArrayList<Procedure> getCompleted() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> return completed;<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span> public int getCompletedCount() {<a name="line.502"></a>
-<span class="sourceLineNo">503</span> return completed.size();<a name="line.503"></a>
-<span class="sourceLineNo">504</span> }<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span> public int getLoadedCount() {<a name="line.506"></a>
-<span class="sourceLineNo">507</span> return runnable.size() + completed.size();<a name="line.507"></a>
-<span class="sourceLineNo">508</span> }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span> public ArrayList<Procedure> getCorrupted() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> return corrupted;<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span> public int getCorruptedCount() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span> return corrupted.size();<a name="line.515"></a>
-<span class="sourceLineNo">516</span> }<a name="line.516"></a>
-<span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span> public boolean isRunnable(final long procId) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span> for (Procedure proc: runnable) {<a name="line.519"></a>
-<span class="sourceLineNo">520</span> if (proc.getProcId() == procId) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> return true;<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return false;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @Override<a name="line.527"></a>
-<span class="sourceLineNo">528</span> public void setMaxProcId(long maxProcId) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> this.maxProcId = maxProcId;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> @Override<a name="line.532"></a>
-<span class="sourceLineNo">533</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.533"></a>
-<span class="sourceLineNo">534</span> while (procIter.hasNext()) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> long procId;<a name="line.535"></a>
-<span class="sourceLineNo">536</span> if (procIter.isNextFinished()) {<a name="line.536"></a>
-<span class="sourceLineNo">537</span> Procedure<?> proc = procIter.next();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> procId = proc.getProcId();<a name="line.538"></a>
-<span class="sourceLineNo">539</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.539"></a>
-<span class="sourceLineNo">540</span> completed.add(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> } else {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> Procedure proc = procIter.next();<a name="line.542"></a>
-<span class="sourceLineNo">543</span> procId = proc.getProcId();<a name="line.543"></a>
-<span class="sourceLineNo">544</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> runnable.add(proc);<a name="line.545"></a>
-<span class="sourceLineNo">546</span> }<a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (procIds != null) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span> }<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> @Override<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.554"></a>
-<span class="sourceLineNo">555</span> while (procIter.hasNext()) {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> Procedure proc = procIter.next();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> corrupted.add(proc);<a name="line.558"></a>
-<span class="sourceLineNo">559</span> }<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span>}<a name="line.562"></a>
+<span class="sourceLineNo">470</span> private Set<Long> procIds;<a name="line.470"></a>
+<span class="sourceLineNo">471</span> private long maxProcId = 0;<a name="line.471"></a>
+<span class="sourceLineNo">472</span><a name="line.472"></a>
+<span class="sourceLineNo">473</span> public LoadCounter() {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> this(null);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> }<a name="line.475"></a>
+<span class="sourceLineNo">476</span><a name="line.476"></a>
+<span class="sourceLineNo">477</span> public LoadCounter(final Set<Long> procIds) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> this.procIds = procIds;<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span> public void reset() {<a name="line.481"></a>
+<span class="sourceLineNo">482</span> reset(null);<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span><a name="line.484"></a>
+<span class="sourceLineNo">485</span> public void reset(final Set<Long> procIds) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> corrupted.clear();<a name="line.486"></a>
+<span class="sourceLineNo">487</span> completed.clear();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> runnable.clear();<a name="line.488"></a>
+<span class="sourceLineNo">489</span> this.procIds = procIds;<a name="line.489"></a>
+<span class="sourceLineNo">490</span> this.maxProcId = 0;<a name="line.490"></a>
+<span class="sourceLineNo">491</span> }<a name="line.491"></a>
+<span class="sourceLineNo">492</span><a name="line.492"></a>
+<span class="sourceLineNo">493</span> public long getMaxProcId() {<a name="line.493"></a>
+<span class="sourceLineNo">494</span> return maxProcId;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span><a name="line.496"></a>
+<span class="sourceLineNo">497</span> public ArrayList<Procedure> getRunnables() {<a name="line.497"></a>
+<span class="sourceLineNo">498</span> return runnable;<a name="line.498"></a>
+<span class="sourceLineNo">499</span> }<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span> public int getRunnableCount() {<a name="line.501"></a>
+<span class="sourceLineNo">502</span> return runnable.size();<a name="line.502"></a>
+<span class="sourceLineNo">503</span> }<a name="line.503"></a>
+<span class="sourceLineNo">504</span><a name="line.504"></a>
+<span class="sourceLineNo">505</span> public ArrayList<Procedure> getCompleted() {<a name="line.505"></a>
+<span class="sourceLineNo">506</span> return completed;<a name="line.506"></a>
+<span class="sourceLineNo">507</span> }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span> public int getCompletedCount() {<a name="line.509"></a>
+<span class="sourceLineNo">510</span> return completed.size();<a name="line.510"></a>
+<span class="sourceLineNo">511</span> }<a name="line.511"></a>
+<span class="sourceLineNo">512</span><a name="line.512"></a>
+<span class="sourceLineNo">513</span> public int getLoadedCount() {<a name="line.513"></a>
+<span class="sourceLineNo">514</span> return runnable.size() + completed.size();<a name="line.514"></a>
+<span class="sourceLineNo">515</span> }<a name="line.515"></a>
+<span class="sourceLineNo">516</span><a name="line.516"></a>
+<span class="sourceLineNo">517</span> public ArrayList<Procedure> getCorrupted() {<a name="line.517"></a>
+<span class="sourceLineNo">518</span> return corrupted;<a name="line.518"></a>
+<span class="sourceLineNo">519</span> }<a name="line.519"></a>
+<span class="sourceLineNo">520</span><a name="line.520"></a>
+<span class="sourceLineNo">521</span> public int getCorruptedCount() {<a name="line.521"></a>
+<span class="sourceLineNo">522</span> return corrupted.size();<a name="line.522"></a>
+<span class="sourceLineNo">523</span> }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span> public boolean isRunnable(final long procId) {<a name="line.525"></a>
+<span class="sourceLineNo">526</span> for (Procedure proc: runnable) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span> if (proc.getProcId() == procId) {<a name="line.527"></a>
+<span class="sourceLineNo">528</span> return true;<a name="line.528"></a>
+<span class="sourceLineNo">529</span> }<a name="line.529"></a>
+<span class="sourceLineNo">530</span> }<a name="line.530"></a>
+<span class="sourceLineNo">531</span> return false;<a name="line.531"></a>
+<span class="sourceLineNo">532</span> }<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span> @Override<a name="line.534"></a>
+<span class="sourceLineNo">535</span> public void setMaxProcId(long maxProcId) {<a name="line.535"></a>
+<span class="sourceLineNo">536</span> this.maxProcId = maxProcId;<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span><a name="line.538"></a>
+<span class="sourceLineNo">539</span> @Override<a name="line.539"></a>
+<span class="sourceLineNo">540</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.540"></a>
+<span class="sourceLineNo">541</span> while (procIter.hasNext()) {<a name="line.541"></a>
+<span class="sourceLineNo">542</span> long procId;<a name="line.542"></a>
+<span class="sourceLineNo">543</span> if (procIter.isNextFinished()) {<a name="line.543"></a>
+<span class="sourceLineNo">544</span> Procedure<?> proc = procIter.next();<a name="line.544"></a>
+<span class="sourceLineNo">545</span> procId = proc.getProcId();<a name="line.545"></a>
+<span class="sourceLineNo">546</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.546"></a>
+<span class="sourceLineNo">547</span> completed.add(proc);<a name="line.547"></a>
+<span class="sourceLineNo">548</span> } else {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> Procedure proc = procIter.next();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> procId = proc.getProcId();<a name="line.550"></a>
+<span class="sourceLineNo">551</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.551"></a>
+<span class="sourceLineNo">552</span> runnable.add(proc);<a name="line.552"></a>
+<span class="sourceLineNo">553</span> }<a name="line.553"></a>
+<span class="sourceLineNo">554</span> if (procIds != null) {<a name="line.554"></a>
+<span class="sourceLineNo">555</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.555"></a>
+<span class="sourceLineNo">556</span> }<a name="line.556"></a>
+<span class="sourceLineNo">557</span> }<a name="line.557"></a>
+<span class="sourceLineNo">558</span> }<a name="line.558"></a>
+<span class="sourceLineNo">559</span><a name="line.559"></a>
+<span class="sourceLineNo">560</span> @Override<a name="line.560"></a>
+<span class="sourceLineNo">561</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.561"></a>
+<span class="sourceLineNo">562</span> while (procIter.hasNext()) {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> Procedure proc = procIter.next();<a name="line.563"></a>
+<span class="sourceLineNo">564</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.564"></a>
+<span class="sourceLineNo">565</span> corrupted.add(proc);<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span> }<a name="line.567"></a>
+<span class="sourceLineNo">568</span> }<a name="line.568"></a>
+<span class="sourceLineNo">569</span>}<a name="line.569"></a>
[20/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
Published site at 1114a1a65e78a00f643ef15b9a13bc710cb80845.
Project: http://git-wip-us.apache.org/repos/asf/hbase-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase-site/commit/3b0a06ec
Tree: http://git-wip-us.apache.org/repos/asf/hbase-site/tree/3b0a06ec
Diff: http://git-wip-us.apache.org/repos/asf/hbase-site/diff/3b0a06ec
Branch: refs/heads/asf-site
Commit: 3b0a06ec3d00787b80edf2a466fe47b4a4ddbac4
Parents: d682c43
Author: jenkins <bu...@apache.org>
Authored: Wed Aug 15 14:47:41 2018 +0000
Committer: jenkins <bu...@apache.org>
Committed: Wed Aug 15 14:47:41 2018 +0000
----------------------------------------------------------------------
acid-semantics.html | 4 +-
apache_hbase_reference_guide.pdf | 4 +-
book.html | 2 +-
bulk-loads.html | 4 +-
checkstyle-aggregate.html | 36 +-
coc.html | 4 +-
dependencies.html | 4 +-
dependency-convergence.html | 4 +-
dependency-info.html | 4 +-
dependency-management.html | 4 +-
devapidocs/constant-values.html | 6 +-
devapidocs/index-all.html | 24 +-
.../hadoop/hbase/backup/package-tree.html | 4 +-
.../hadoop/hbase/client/package-tree.html | 22 +-
.../hadoop/hbase/coprocessor/package-tree.html | 2 +-
.../hadoop/hbase/filter/package-tree.html | 10 +-
.../hadoop/hbase/io/hfile/package-tree.html | 4 +-
.../hadoop/hbase/mapreduce/package-tree.html | 4 +-
.../hbase/master/balancer/package-tree.html | 2 +-
.../hadoop/hbase/master/package-tree.html | 6 +-
.../hbase/master/procedure/package-tree.html | 2 +-
.../org/apache/hadoop/hbase/package-tree.html | 14 +-
...edureExecutor.CompletedProcedureCleaner.html | 32 +-
...dureExecutor.CompletedProcedureRetainer.html | 18 +-
.../ProcedureExecutor.FailedProcedure.html | 20 +-
...ProcedureExecutor.KeepAliveWorkerThread.html | 6 +-
...edureExecutor.ProcedureExecutorListener.html | 8 +-
.../procedure2/ProcedureExecutor.Testing.html | 86 +-
.../ProcedureExecutor.WorkerMonitor.html | 32 +-
.../ProcedureExecutor.WorkerThread.html | 20 +-
.../hbase/procedure2/ProcedureExecutor.html | 262 +-
.../class-use/ProcedureExecutor.Testing.html | 4 +-
.../hbase/procedure2/package-summary.html | 4 +-
.../hadoop/hbase/procedure2/package-tree.html | 4 +-
.../hadoop/hbase/procedure2/package-use.html | 4 +-
.../hadoop/hbase/quotas/package-tree.html | 4 +-
.../hadoop/hbase/regionserver/package-tree.html | 18 +-
.../regionserver/querymatcher/package-tree.html | 2 +-
.../hbase/regionserver/wal/package-tree.html | 4 +-
.../hadoop/hbase/rest/model/package-tree.html | 2 +-
.../hadoop/hbase/security/package-tree.html | 2 +-
.../hadoop/hbase/thrift/package-tree.html | 2 +-
.../apache/hadoop/hbase/util/package-tree.html | 8 +-
.../apache/hadoop/hbase/wal/package-tree.html | 2 +-
.../org/apache/hadoop/hbase/Version.html | 6 +-
...edureExecutor.CompletedProcedureCleaner.html | 3685 +++++++++---------
...dureExecutor.CompletedProcedureRetainer.html | 3685 +++++++++---------
.../ProcedureExecutor.FailedProcedure.html | 3685 +++++++++---------
...ProcedureExecutor.KeepAliveWorkerThread.html | 3685 +++++++++---------
...edureExecutor.ProcedureExecutorListener.html | 3685 +++++++++---------
.../procedure2/ProcedureExecutor.Testing.html | 3685 +++++++++---------
.../ProcedureExecutor.WorkerMonitor.html | 3685 +++++++++---------
.../ProcedureExecutor.WorkerThread.html | 3685 +++++++++---------
.../hbase/procedure2/ProcedureExecutor.html | 3685 +++++++++---------
downloads.html | 4 +-
export_control.html | 4 +-
index.html | 4 +-
integration.html | 4 +-
issue-tracking.html | 4 +-
license.html | 4 +-
mail-lists.html | 4 +-
metrics.html | 4 +-
old_news.html | 4 +-
plugin-management.html | 4 +-
plugins.html | 4 +-
poweredbyhbase.html | 4 +-
project-info.html | 4 +-
project-reports.html | 4 +-
project-summary.html | 4 +-
pseudo-distributed.html | 4 +-
replication.html | 4 +-
resources.html | 4 +-
source-repository.html | 4 +-
sponsors.html | 4 +-
supportingprojects.html | 4 +-
team-list.html | 4 +-
testdevapidocs/index-all.html | 9 +
.../hadoop/hbase/io/hfile/package-tree.html | 2 +-
.../org/apache/hadoop/hbase/package-tree.html | 10 +-
.../ProcedureTestingUtility.LoadCounter.html | 44 +-
.../ProcedureTestingUtility.NoopProcedure.html | 14 +-
.../ProcedureTestingUtility.TestProcedure.html | 28 +-
.../procedure2/ProcedureTestingUtility.html | 71 +-
.../TestChildProcedures.TestChildProcedure.html | 10 +-
.../TestChildProcedures.TestProcEnv.html | 23 +-
.../TestChildProcedures.TestRootProcedure.html | 10 +-
.../hbase/procedure2/TestChildProcedures.html | 36 +-
.../hadoop/hbase/procedure2/package-tree.html | 4 +-
.../hadoop/hbase/regionserver/package-tree.html | 6 +-
.../apache/hadoop/hbase/test/package-tree.html | 4 +-
.../apache/hadoop/hbase/wal/package-tree.html | 2 +-
.../ProcedureTestingUtility.LoadCounter.html | 755 ++--
.../ProcedureTestingUtility.NoopProcedure.html | 755 ++--
.../ProcedureTestingUtility.TestProcedure.html | 755 ++--
.../procedure2/ProcedureTestingUtility.html | 755 ++--
.../TestChildProcedures.TestChildProcedure.html | 199 +-
.../TestChildProcedures.TestProcEnv.html | 199 +-
.../TestChildProcedures.TestRootProcedure.html | 199 +-
.../hbase/procedure2/TestChildProcedures.html | 199 +-
99 files changed, 19438 insertions(+), 18624 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/acid-semantics.html
----------------------------------------------------------------------
diff --git a/acid-semantics.html b/acid-semantics.html
index b3c6804..f680c8b 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Apache HBase (TM) ACID Properties
@@ -601,7 +601,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/apache_hbase_reference_guide.pdf
----------------------------------------------------------------------
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index 52ff7ac..60cdac9 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,8 +5,8 @@
/Author (Apache HBase Team)
/Creator (Asciidoctor PDF 1.5.0.alpha.15, based on Prawn 2.2.2)
/Producer (Apache HBase Team)
-/ModDate (D:20180814142952+00'00')
-/CreationDate (D:20180814144340+00'00')
+/ModDate (D:20180815143043+00'00')
+/CreationDate (D:20180815144429+00'00')
>>
endobj
2 0 obj
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/book.html
----------------------------------------------------------------------
diff --git a/book.html b/book.html
index d9add12..8b4dfc5 100644
--- a/book.html
+++ b/book.html
@@ -41148,7 +41148,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/
<div id="footer">
<div id="footer-text">
Version 3.0.0-SNAPSHOT<br>
-Last updated 2018-08-14 14:29:52 UTC
+Last updated 2018-08-15 14:30:43 UTC
</div>
</div>
</body>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/bulk-loads.html
----------------------------------------------------------------------
diff --git a/bulk-loads.html b/bulk-loads.html
index 91d70b6..a10609d 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Bulk Loads in Apache HBase (TM)
@@ -306,7 +306,7 @@ under the License. -->
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/checkstyle-aggregate.html
----------------------------------------------------------------------
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index fd94e87..cdd3551 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Checkstyle Results</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -9883,12 +9883,12 @@
<td><a class="externalLink" href="http://checkstyle.sourceforge.net/config_javadoc.html#JavadocTagContinuationIndentation">JavadocTagContinuationIndentation</a>
<ul>
<li>offset: <tt>"2"</tt></li></ul></td>
-<td>771</td>
+<td>770</td>
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td></tr>
<tr class="b">
<td></td>
<td><a class="externalLink" href="http://checkstyle.sourceforge.net/config_javadoc.html#NonEmptyAtclauseDescription">NonEmptyAtclauseDescription</a></td>
-<td>3618</td>
+<td>3619</td>
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td></tr>
<tr class="a">
<td>misc</td>
@@ -19524,7 +19524,7 @@
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
-<td>JavadocTagContinuationIndentation</td>
+<td>NonEmptyAtclauseDescription</td>
<td>Javadoc comment at column 0 has parse error. Details: no viable alternative at input ' *' while parsing JAVADOC_TAG</td>
<td>117</td></tr>
<tr class="a">
@@ -73417,62 +73417,62 @@
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>sizes</td>
<td>MethodLength</td>
-<td>Method length is 151 lines (max allowed is 150).</td>
-<td>418</td></tr>
+<td>Method length is 162 lines (max allowed is 150).</td>
+<td>453</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>535</td></tr>
+<td>581</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>541</td></tr>
+<td>587</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>767</td></tr>
+<td>813</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>javadoc</td>
<td>NonEmptyAtclauseDescription</td>
<td>At-clause should have a non-empty description.</td>
-<td>768</td></tr>
+<td>814</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>787</td></tr>
+<td>833</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>796</td></tr>
+<td>842</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>818</td></tr>
+<td>864</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>821</td></tr>
+<td>867</td></tr>
<tr class="a">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>1208</td></tr></table></div>
+<td>1254</td></tr></table></div>
<div class="section">
<h3 id="org.apache.hadoop.hbase.procedure2.ProcedureScheduler.java">org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java</h3>
<table border="0" class="table table-striped">
@@ -73514,13 +73514,13 @@
<td>blocks</td>
<td>NeedBraces</td>
<td>'if' construct must use '{}'s.</td>
-<td>179</td></tr>
+<td>186</td></tr>
<tr class="b">
<td><img src="images/icon_error_sml.gif" alt="" /> Error</td>
<td>blocks</td>
<td>LeftCurly</td>
<td>'{' at column 39 should have line break after.</td>
-<td>372</td></tr></table></div>
+<td>379</td></tr></table></div>
<div class="section">
<h3 id="org.apache.hadoop.hbase.procedure2.ProcedureUtil.java">org/apache/hadoop/hbase/procedure2/ProcedureUtil.java</h3>
<table border="0" class="table table-striped">
@@ -120351,7 +120351,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/coc.html
----------------------------------------------------------------------
diff --git a/coc.html b/coc.html
index b2b40c4..0a56ebd 100644
--- a/coc.html
+++ b/coc.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase –
Code of Conduct Policy
@@ -375,7 +375,7 @@ email to <a class="externalLink" href="mailto:private@hbase.apache.org">the priv
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/dependencies.html
----------------------------------------------------------------------
diff --git a/dependencies.html b/dependencies.html
index 6db77d9..2052dc8 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Dependencies</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -440,7 +440,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/dependency-convergence.html
----------------------------------------------------------------------
diff --git a/dependency-convergence.html b/dependency-convergence.html
index d108e43..b83d6fc 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Reactor Dependency Convergence</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -890,7 +890,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/dependency-info.html
----------------------------------------------------------------------
diff --git a/dependency-info.html b/dependency-info.html
index 111a518..618decf 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Dependency Information</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -313,7 +313,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/dependency-management.html
----------------------------------------------------------------------
diff --git a/dependency-management.html b/dependency-management.html
index 53cae18..1061be4 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -7,7 +7,7 @@
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
- <meta name="Date-Revision-yyyymmdd" content="20180814" />
+ <meta name="Date-Revision-yyyymmdd" content="20180815" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache HBase – Project Dependency Management</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.4-HBase.min.css" />
@@ -1005,7 +1005,7 @@
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
All rights reserved.
- <li id="publishDate" class="pull-right">Last Published: 2018-08-14</li>
+ <li id="publishDate" class="pull-right">Last Published: 2018-08-15</li>
</p>
</div>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/constant-values.html
----------------------------------------------------------------------
diff --git a/devapidocs/constant-values.html b/devapidocs/constant-values.html
index e0622a4..d2ec4af 100644
--- a/devapidocs/constant-values.html
+++ b/devapidocs/constant-values.html
@@ -3817,21 +3817,21 @@
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#date">date</a></code></td>
-<td class="colLast"><code>"Tue Aug 14 14:38:23 UTC 2018"</code></td>
+<td class="colLast"><code>"Wed Aug 15 14:39:17 UTC 2018"</code></td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.revision">
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#revision">revision</a></code></td>
-<td class="colLast"><code>"e705cf1447dd7e44f0cc0297746a0f14135e2f2f"</code></td>
+<td class="colLast"><code>"1114a1a65e78a00f643ef15b9a13bc710cb80845"</code></td>
</tr>
<tr class="altColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.srcChecksum">
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#srcChecksum">srcChecksum</a></code></td>
-<td class="colLast"><code>"0d84c58b6fa1860ba76507b3d036fb9b"</code></td>
+<td class="colLast"><code>"b03e26b326c47068ee3c2d38b4181ca8"</code></td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.url">
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/index-all.html
----------------------------------------------------------------------
diff --git a/devapidocs/index-all.html b/devapidocs/index-all.html
index 16e8bbc..50c58f5 100644
--- a/devapidocs/index-all.html
+++ b/devapidocs/index-all.html
@@ -65949,10 +65949,18 @@
</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/KeyValueUtil.html#KeyValueUtil--">KeyValueUtil()</a></span> - Constructor for class org.apache.hadoop.hbase.<a href="org/apache/hadoop/hbase/KeyValueUtil.html" title="class in org.apache.hadoop.hbase">KeyValueUtil</a></dt>
<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#kill-java.lang.String-">kill(String)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor</a></dt>
+<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/HRegionServer.html#kill--">kill()</a></span> - Method in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/HRegionServer.html" title="class in org.apache.hadoop.hbase.regionserver">HRegionServer</a></dt>
<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#killAfterStoreUpdate">killAfterStoreUpdate</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
+<dd>
+<div class="block">Set when we want to fail AFTER state has been stored into the WAL.</div>
+</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#killBeforeStoreUpdate">killBeforeStoreUpdate</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
-<dd> </dd>
+<dd>
+<div class="block">Kill the PE BEFORE we store state to the WAL.</div>
+</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.html#killCurrentActiveZooKeeperServer--">killCurrentActiveZooKeeperServer()</a></span> - Method in class org.apache.hadoop.hbase.zookeeper.<a href="org/apache/hadoop/hbase/zookeeper/MiniZooKeeperCluster.html" title="class in org.apache.hadoop.hbase.zookeeper">MiniZooKeeperCluster</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/regionserver/HRegionServer.html#killed">killed</a></span> - Variable in class org.apache.hadoop.hbase.regionserver.<a href="org/apache/hadoop/hbase/regionserver/HRegionServer.html" title="class in org.apache.hadoop.hbase.regionserver">HRegionServer</a></dt>
@@ -85625,7 +85633,9 @@ service.</div>
<dt><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">ProcedureExecutor.ProcedureExecutorListener</span></a> - Interface in <a href="org/apache/hadoop/hbase/procedure2/package-summary.html">org.apache.hadoop.hbase.procedure2</a></dt>
<dd> </dd>
<dt><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">ProcedureExecutor.Testing</span></a> - Class in <a href="org/apache/hadoop/hbase/procedure2/package-summary.html">org.apache.hadoop.hbase.procedure2</a></dt>
-<dd> </dd>
+<dd>
+<div class="block">Class with parameters describing how to fail/die when in testing-context.</div>
+</dd>
<dt><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html" title="class in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">ProcedureExecutor.WorkerMonitor</span></a> - Class in <a href="org/apache/hadoop/hbase/procedure2/package-summary.html">org.apache.hadoop.hbase.procedure2</a></dt>
<dd> </dd>
<dt><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">ProcedureExecutor.WorkerThread</span></a> - Class in <a href="org/apache/hadoop/hbase/procedure2/package-summary.html">org.apache.hadoop.hbase.procedure2</a></dt>
@@ -105343,6 +105353,10 @@ service.</div>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.html#shouldIncludeMemStoreTS--">shouldIncludeMemStoreTS()</a></span> - Method in class org.apache.hadoop.hbase.io.hfile.<a href="org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.html" title="class in org.apache.hadoop.hbase.io.hfile">HFileReaderImpl</a></dt>
<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillAfterStoreUpdate--">shouldKillAfterStoreUpdate()</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
+<dd> </dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillAfterStoreUpdate-boolean-">shouldKillAfterStoreUpdate(boolean)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
+<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillBeforeStoreUpdate--">shouldKillBeforeStoreUpdate()</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillBeforeStoreUpdate-boolean-">shouldKillBeforeStoreUpdate(boolean)</a></span> - Method in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
@@ -112998,7 +113012,9 @@ service.</div>
an exception.</div>
</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing">testing</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor</a></dt>
-<dd> </dd>
+<dd>
+<div class="block"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing"><code>ProcedureExecutor.testing</code></a> is non-null when ProcedureExecutor is being tested.</div>
+</dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#Testing--">Testing()</a></span> - Constructor for class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportMapper.html#testing">testing</a></span> - Static variable in class org.apache.hadoop.hbase.snapshot.<a href="org/apache/hadoop/hbase/snapshot/ExportSnapshot.ExportMapper.html" title="class in org.apache.hadoop.hbase.snapshot">ExportSnapshot.ExportMapper</a></dt>
@@ -114378,6 +114394,8 @@ service.</div>
<dd>
<div class="block">Presumes float encoded as IEEE 754 floating-point "single format"</div>
</dd>
+<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#toggleKillAfterStoreUpdate">toggleKillAfterStoreUpdate</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
+<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#toggleKillBeforeStoreUpdate">toggleKillBeforeStoreUpdate</a></span> - Variable in class org.apache.hadoop.hbase.procedure2.<a href="org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></dt>
<dd> </dd>
<dt><span class="memberNameLink"><a href="org/apache/hadoop/hbase/AuthUtil.html#toGroupEntry-java.lang.String-">toGroupEntry(String)</a></span> - Static method in class org.apache.hadoop.hbase.<a href="org/apache/hadoop/hbase/AuthUtil.html" title="class in org.apache.hadoop.hbase">AuthUtil</a></dt>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
index f55f5e6..38f7cc7 100644
--- a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
@@ -167,10 +167,10 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupState.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupPhase.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupPhase</span></a></li>
<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupType.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupRestoreConstants.BackupCommand.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupRestoreConstants.BackupCommand</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupPhase.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupPhase</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupState.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupState</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/client/package-tree.html b/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
index 7277664..1255e0b 100644
--- a/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
@@ -552,23 +552,23 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RequestController.ReturnCode.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RequestController.ReturnCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/ScannerCallable.MoreResults.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">ScannerCallable.MoreResults</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RegionLocateType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RegionLocateType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MasterSwitchType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MasterSwitchType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AbstractResponse.ResponseType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AbstractResponse.ResponseType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/SnapshotType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">SnapshotType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Scan.ReadType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Scan.ReadType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.Retry.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncRequestFutureImpl.Retry</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MobCompactPartitionPolicy</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactionState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactionState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncProcessTask.SubmittedRows.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncProcessTask.SubmittedRows</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/TableState.State.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">TableState.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RequestController.ReturnCode.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RequestController.ReturnCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Durability.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Durability</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/SnapshotType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">SnapshotType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/TableState.State.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">TableState.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.Retry.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncRequestFutureImpl.Retry</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AbstractResponse.ResponseType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AbstractResponse.ResponseType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactionState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactionState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/IsolationLevel.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">IsolationLevel</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MasterSwitchType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MasterSwitchType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MobCompactPartitionPolicy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Consistency.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Consistency</span></a></li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html b/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
index 329f62f..e67b778 100644
--- a/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
@@ -201,8 +201,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/RegionObserver.MutationType.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">RegionObserver.MutationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.MetaTableOps.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">MetaTableMetrics.MetaTableOps</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/RegionObserver.MutationType.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">RegionObserver.MutationType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html b/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
index aa61321..7e5f48f 100644
--- a/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
@@ -183,14 +183,14 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/CompareFilter.CompareOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">CompareFilter.CompareOp</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/RegexStringComparator.EngineType.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">RegexStringComparator.EngineType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/Filter.ReturnCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">Filter.ReturnCode</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterList.Operator.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterList.Operator</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterWrapper.FilterRowRetCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterWrapper.FilterRowRetCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.SatisfiesCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.SatisfiesCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.Order.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.Order</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterList.Operator.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterList.Operator</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/BitComparator.BitwiseOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">BitComparator.BitwiseOp</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/CompareFilter.CompareOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">CompareFilter.CompareOp</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.SatisfiesCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.SatisfiesCode</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterWrapper.FilterRowRetCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterWrapper.FilterRowRetCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/Filter.ReturnCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">Filter.ReturnCode</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html b/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
index a41bce3..8e30386 100644
--- a/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
@@ -274,12 +274,12 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockPriority.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockPriority</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/Cacheable.MemoryType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">Cacheable.MemoryType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.BlockCategory.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType.BlockCategory</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/HFileBlock.Writer.State.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">HFileBlock.Writer.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/Cacheable.MemoryType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">Cacheable.MemoryType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/CacheConfig.ExternalBlockCaches.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">CacheConfig.ExternalBlockCaches</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html b/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
index 76f6d8f..bd6fd7e 100644
--- a/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
@@ -293,10 +293,10 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/CellCounter.CellCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">CellCounter.CellCounterMapper.Counters</span></a></li>
<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/RowCounter.RowCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">RowCounter.RowCounterMapper.Counters</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/TableSplit.Version.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">TableSplit.Version</span></a></li>
<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/SyncTable.SyncMapper.Counter.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">SyncTable.SyncMapper.Counter</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/TableSplit.Version.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">TableSplit.Version</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/CellCounter.CellCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">CellCounter.CellCounterMapper.Counters</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
index abd92a9..1799704 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/balancer/package-tree.html
@@ -198,8 +198,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.master.balancer.<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer"><span class="typeNameLink">BaseLoadBalancer.Cluster.LocalityType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.balancer.<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.Action.Type.html" title="enum in org.apache.hadoop.hbase.master.balancer"><span class="typeNameLink">BaseLoadBalancer.Cluster.Action.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.balancer.<a href="../../../../../../org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.Cluster.LocalityType.html" title="enum in org.apache.hadoop.hbase.master.balancer"><span class="typeNameLink">BaseLoadBalancer.Cluster.LocalityType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
index 4c308e0..2a6bd56 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
@@ -348,11 +348,11 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.TerminationStatus</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.ResubmitDirective</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MetricsMasterSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MetricsMasterSourceFactoryImpl.FactoryStorage</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MasterRpcServices.BalanceSwitchMode.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MasterRpcServices.BalanceSwitchMode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/RegionState.State.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">RegionState.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.ResubmitDirective</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.TerminationStatus</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MetricsMasterSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MetricsMasterSourceFactoryImpl.FactoryStorage</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
index 8344765..ddea7b8 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
@@ -216,9 +216,9 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
+<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/MetaProcedureInterface.MetaOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">MetaProcedureInterface.MetaOperationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.PeerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">PeerProcedureInterface.PeerOperationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.ServerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">ServerProcedureInterface.ServerOperationType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/MetaProcedureInterface.MetaOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">MetaProcedureInterface.MetaOperationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.TableOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">TableProcedureInterface.TableOperationType</span></a></li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/package-tree.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/package-tree.html b/devapidocs/org/apache/hadoop/hbase/package-tree.html
index 92c3845..6abb0e4 100644
--- a/devapidocs/org/apache/hadoop/hbase/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/package-tree.html
@@ -437,19 +437,19 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true" title="class or interface in java.io">Serializable</a>)
<ul>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompareOperator.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompareOperator</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Coprocessor.State.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Coprocessor.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MemoryCompactionPolicy.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MemoryCompactionPolicy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompatibilitySingletonFactory.SingletonStorage.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompatibilitySingletonFactory.SingletonStorage</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeyValue.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeyValue.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HealthChecker.HealthCheckerExitStatus.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HealthChecker.HealthCheckerExitStatus</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HConstants.OperationStatusCode.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HConstants.OperationStatusCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ClusterMetrics.Option</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MemoryCompactionPolicy.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MemoryCompactionPolicy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Cell.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Cell.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompareOperator.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompareOperator</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Coprocessor.State.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Coprocessor.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MetaTableAccessor.QueryType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MetaTableAccessor.QueryType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Size.Unit.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Size.Unit</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CellBuilderType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CellBuilderType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HealthChecker.HealthCheckerExitStatus.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HealthChecker.HealthCheckerExitStatus</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeyValue.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeyValue.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HConstants.OperationStatusCode.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HConstants.OperationStatusCode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeepDeletedCells.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeepDeletedCells</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MetaTableAccessor.QueryType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MetaTableAccessor.QueryType</span></a></li>
</ul>
</li>
</ul>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
index 81c701f..9d2fc1d 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html
@@ -127,7 +127,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.160">ProcedureExecutor.CompletedProcedureCleaner</a><TEnvironment>
+<pre>private static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.195">ProcedureExecutor.CompletedProcedureCleaner</a><TEnvironment>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><TEnvironment></pre>
<div class="block">Internal cleaner that removes the completed procedure results after a TTL.
NOTE: This is a special case handled in timeoutLoop().
@@ -317,7 +317,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>LOG</h4>
-<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.162">LOG</a></pre>
+<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.197">LOG</a></pre>
</li>
</ul>
<a name="CLEANER_INTERVAL_CONF_KEY">
@@ -326,7 +326,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>CLEANER_INTERVAL_CONF_KEY</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.164">CLEANER_INTERVAL_CONF_KEY</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.199">CLEANER_INTERVAL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.CLEANER_INTERVAL_CONF_KEY">Constant Field Values</a></dd>
@@ -339,7 +339,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_CLEANER_INTERVAL</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.165">DEFAULT_CLEANER_INTERVAL</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.200">DEFAULT_CLEANER_INTERVAL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.DEFAULT_CLEANER_INTERVAL">Constant Field Values</a></dd>
@@ -352,7 +352,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>EVICT_TTL_CONF_KEY</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.167">EVICT_TTL_CONF_KEY</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.202">EVICT_TTL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.EVICT_TTL_CONF_KEY">Constant Field Values</a></dd>
@@ -365,7 +365,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_EVICT_TTL</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.168">DEFAULT_EVICT_TTL</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.203">DEFAULT_EVICT_TTL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.DEFAULT_EVICT_TTL">Constant Field Values</a></dd>
@@ -378,7 +378,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>EVICT_ACKED_TTL_CONF_KEY</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.170">EVICT_ACKED_TTL_CONF_KEY</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.205">EVICT_ACKED_TTL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.EVICT_ACKED_TTL_CONF_KEY">Constant Field Values</a></dd>
@@ -391,7 +391,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_ACKED_EVICT_TTL</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.171">DEFAULT_ACKED_EVICT_TTL</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.206">DEFAULT_ACKED_EVICT_TTL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.DEFAULT_ACKED_EVICT_TTL">Constant Field Values</a></dd>
@@ -404,7 +404,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>BATCH_SIZE_CONF_KEY</h4>
-<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.173">BATCH_SIZE_CONF_KEY</a></pre>
+<pre>private static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.208">BATCH_SIZE_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.BATCH_SIZE_CONF_KEY">Constant Field Values</a></dd>
@@ -417,7 +417,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_BATCH_SIZE</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.174">DEFAULT_BATCH_SIZE</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.209">DEFAULT_BATCH_SIZE</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CompletedProcedureCleaner.DEFAULT_BATCH_SIZE">Constant Field Values</a></dd>
@@ -430,7 +430,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>completed</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.CompletedProcedureRetainer</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html" title="type parameter in ProcedureExecutor.CompletedProcedureCleaner">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.176">completed</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.CompletedProcedureRetainer</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html" title="type parameter in ProcedureExecutor.CompletedProcedureCleaner">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.211">completed</a></pre>
</li>
</ul>
<a name="nonceKeysToProcIdsMap">
@@ -439,7 +439,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>nonceKeysToProcIdsMap</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.177">nonceKeysToProcIdsMap</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.212">nonceKeysToProcIdsMap</a></pre>
</li>
</ul>
<a name="store">
@@ -448,7 +448,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockList">
<li class="blockList">
<h4>store</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.178">store</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.213">store</a></pre>
</li>
</ul>
<a name="conf">
@@ -457,7 +457,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockListLast">
<li class="blockList">
<h4>conf</h4>
-<pre>private org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.179">conf</a></pre>
+<pre>private org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.214">conf</a></pre>
</li>
</ul>
</li>
@@ -474,7 +474,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockListLast">
<li class="blockList">
<h4>CompletedProcedureCleaner</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.181">CompletedProcedureCleaner</a>(org.apache.hadoop.conf.Configuration conf,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.216">CompletedProcedureCleaner</a>(org.apache.hadoop.conf.Configuration conf,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> store,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.CompletedProcedureRetainer</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html" title="type parameter in ProcedureExecutor.CompletedProcedureCleaner">TEnvironment</a>>> completedMap,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Map.html?is-external=true" title="class or interface in java.util">Map</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> nonceKeysToProcIdsMap)</pre>
@@ -496,7 +496,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMe
<ul class="blockListLast">
<li class="blockList">
<h4>periodicExecute</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.193">periodicExecute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html" title="type parameter in ProcedureExecutor.CompletedProcedureCleaner">TEnvironment</a> env)</pre>
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html#line.228">periodicExecute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html" title="type parameter in ProcedureExecutor.CompletedProcedureCleaner">TEnvironment</a> env)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html#periodicExecute-TEnvironment-">periodicExecute</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureCleaner.html" title="type parameter in ProcedureExecutor.CompletedProcedureCleaner">TEnvironment</a>></code></dd>
[10/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span class="sourceLineNo">600</span><a name="line.600"></a>
-<span
<TRUNCATED>
[19/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
index e293605..ed26123 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html
@@ -113,7 +113,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private static final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.112">ProcedureExecutor.CompletedProcedureRetainer</a><TEnvironment>
+<pre>private static final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.147">ProcedureExecutor.CompletedProcedureRetainer</a><TEnvironment>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
</li>
</ul>
@@ -223,7 +223,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>procedure</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="type parameter in ProcedureExecutor.CompletedProcedureRetainer">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.113">procedure</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="type parameter in ProcedureExecutor.CompletedProcedureRetainer">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.148">procedure</a></pre>
</li>
</ul>
<a name="clientAckTime">
@@ -232,7 +232,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>clientAckTime</h4>
-<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.114">clientAckTime</a></pre>
+<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.149">clientAckTime</a></pre>
</li>
</ul>
</li>
@@ -249,7 +249,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>CompletedProcedureRetainer</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.116">CompletedProcedureRetainer</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="type parameter in ProcedureExecutor.CompletedProcedureRetainer">TEnvironment</a>> procedure)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.151">CompletedProcedureRetainer</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="type parameter in ProcedureExecutor.CompletedProcedureRetainer">TEnvironment</a>> procedure)</pre>
</li>
</ul>
</li>
@@ -266,7 +266,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedure</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="type parameter in ProcedureExecutor.CompletedProcedureRetainer">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.121">getProcedure</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="type parameter in ProcedureExecutor.CompletedProcedureRetainer">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.156">getProcedure</a>()</pre>
</li>
</ul>
<a name="hasClientAckTime--">
@@ -275,7 +275,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>hasClientAckTime</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.125">hasClientAckTime</a>()</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.160">hasClientAckTime</a>()</pre>
</li>
</ul>
<a name="getClientAckTime--">
@@ -284,7 +284,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getClientAckTime</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.129">getClientAckTime</a>()</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.164">getClientAckTime</a>()</pre>
</li>
</ul>
<a name="setClientAckTime-long-">
@@ -293,7 +293,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setClientAckTime</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.133">setClientAckTime</a>(long clientAckTime)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.168">setClientAckTime</a>(long clientAckTime)</pre>
</li>
</ul>
<a name="isExpired-long-long-long-">
@@ -302,7 +302,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>isExpired</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.137">isExpired</a>(long now,
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html#line.172">isExpired</a>(long now,
long evictTtl,
long evictAckTtl)</pre>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
index c8238a4..847ee6a 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html
@@ -122,7 +122,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.829">ProcedureExecutor.FailedProcedure</a><TEnvironment>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.875">ProcedureExecutor.FailedProcedure</a><TEnvironment>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><TEnvironment></pre>
</li>
</ul>
@@ -279,7 +279,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockListLast">
<li class="blockList">
<h4>procName</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.830">procName</a></pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.876">procName</a></pre>
</li>
</ul>
</li>
@@ -296,7 +296,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockList">
<li class="blockList">
<h4>FailedProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.832">FailedProcedure</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.878">FailedProcedure</a>()</pre>
</li>
</ul>
<a name="FailedProcedure-long-java.lang.String-org.apache.hadoop.hbase.security.User-org.apache.hadoop.hbase.util.NonceKey-java.io.IOException-">
@@ -305,7 +305,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockListLast">
<li class="blockList">
<h4>FailedProcedure</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.835">FailedProcedure</a>(long procId,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.881">FailedProcedure</a>(long procId,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> procName,
<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a> owner,
<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey,
@@ -326,7 +326,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockList">
<li class="blockList">
<h4>getProcName</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.849">getProcName</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.895">getProcName</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html#getProcName--">getProcName</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a>></code></dd>
@@ -341,7 +341,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockList">
<li class="blockList">
<h4>execute</h4>
-<pre>protected <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a>>[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.854">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a> env)
+<pre>protected <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a>>[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.900">execute</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a> env)
throws <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureYieldException.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureYieldException</a>,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureSuspendedException.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureSuspendedException</a>,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a></pre>
@@ -373,7 +373,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockList">
<li class="blockList">
<h4>rollback</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.861">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a> env)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.907">rollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a> env)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a>,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a></pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html#rollback-TEnvironment-">Procedure</a></code></span></div>
@@ -402,7 +402,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockList">
<li class="blockList">
<h4>abort</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.867">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a> env)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.913">abort</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html" title="type parameter in ProcedureExecutor.FailedProcedure">TEnvironment</a> env)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html#abort-TEnvironment-">Procedure</a></code></span></div>
<div class="block">The abort() call is asynchronous and each procedure must decide how to deal
with it, if they want to be abortable. The simplest implementation
@@ -425,7 +425,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockList">
<li class="blockList">
<h4>serializeStateData</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.872">serializeStateData</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureStateSerializer.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureStateSerializer</a> serializer)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.918">serializeStateData</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureStateSerializer.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureStateSerializer</a> serializer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html#serializeStateData-org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer-">Procedure</a></code></span></div>
<div class="block">The user-level code of the procedure may have some state to
@@ -447,7 +447,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.htm
<ul class="blockListLast">
<li class="blockList">
<h4>deserializeStateData</h4>
-<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.877">deserializeStateData</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureStateSerializer.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureStateSerializer</a> serializer)
+<pre>protected void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.FailedProcedure.html#line.923">deserializeStateData</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureStateSerializer.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureStateSerializer</a> serializer)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from class: <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html#deserializeStateData-org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer-">Procedure</a></code></span></div>
<div class="block">Called on store load to allow the user to decode the previously serialized
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
index bdb3199..c379429 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
@@ -132,7 +132,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1841">ProcedureExecutor.KeepAliveWorkerThread</a>
+<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1898">ProcedureExecutor.KeepAliveWorkerThread</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a></pre>
</li>
</ul>
@@ -252,7 +252,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExec
<ul class="blockListLast">
<li class="blockList">
<h4>KeepAliveWorkerThread</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.1843">KeepAliveWorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.1900">KeepAliveWorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
</li>
</ul>
</li>
@@ -269,7 +269,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExec
<ul class="blockListLast">
<li class="blockList">
<h4>keepAlive</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.1848">keepAlive</a>(long lastUpdate)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.1905">keepAlive</a>(long lastUpdate)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#keepAlive-long-">keepAlive</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a></code></dd>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
index b136d34..56d4b1e 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
@@ -105,7 +105,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.106">ProcedureExecutor.ProcedureExecutorListener</a></pre>
+<pre>public static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.141">ProcedureExecutor.ProcedureExecutorListener</a></pre>
</li>
</ul>
</div>
@@ -157,7 +157,7 @@ var activeTableTab = "activeTableTab";
<ul class="blockList">
<li class="blockList">
<h4>procedureLoaded</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.107">procedureLoaded</a>(long procId)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.142">procedureLoaded</a>(long procId)</pre>
</li>
</ul>
<a name="procedureAdded-long-">
@@ -166,7 +166,7 @@ var activeTableTab = "activeTableTab";
<ul class="blockList">
<li class="blockList">
<h4>procedureAdded</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.108">procedureAdded</a>(long procId)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.143">procedureAdded</a>(long procId)</pre>
</li>
</ul>
<a name="procedureFinished-long-">
@@ -175,7 +175,7 @@ var activeTableTab = "activeTableTab";
<ul class="blockListLast">
<li class="blockList">
<h4>procedureFinished</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.109">procedureFinished</a>(long procId)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.144">procedureFinished</a>(long procId)</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
index a0d9b1a..9ec6338 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
@@ -18,7 +18,7 @@
catch(err) {
}
//-->
-var methods = {"i0":10,"i1":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -113,8 +113,9 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.87">ProcedureExecutor.Testing</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.96">ProcedureExecutor.Testing</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
+<div class="block">Class with parameters describing how to fail/die when in testing-context.</div>
</li>
</ul>
</div>
@@ -135,12 +136,24 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr class="altColor">
<td class="colFirst"><code>protected boolean</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#killBeforeStoreUpdate">killBeforeStoreUpdate</a></span></code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#killAfterStoreUpdate">killAfterStoreUpdate</a></span></code>
+<div class="block">Set when we want to fail AFTER state has been stored into the WAL.</div>
+</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>protected boolean</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#killBeforeStoreUpdate">killBeforeStoreUpdate</a></span></code>
+<div class="block">Kill the PE BEFORE we store state to the WAL.</div>
+</td>
+</tr>
+<tr class="altColor">
+<td class="colFirst"><code>protected boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#killIfSuspended">killIfSuspended</a></span></code> </td>
</tr>
+<tr class="rowColor">
+<td class="colFirst"><code>protected boolean</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#toggleKillAfterStoreUpdate">toggleKillAfterStoreUpdate</a></span></code> </td>
+</tr>
<tr class="altColor">
<td class="colFirst"><code>protected boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#toggleKillBeforeStoreUpdate">toggleKillBeforeStoreUpdate</a></span></code> </td>
@@ -179,10 +192,18 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>protected boolean</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillBeforeStoreUpdate--">shouldKillBeforeStoreUpdate</a></span>()</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillAfterStoreUpdate--">shouldKillAfterStoreUpdate</a></span>()</code> </td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>protected boolean</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillAfterStoreUpdate-boolean-">shouldKillAfterStoreUpdate</a></span>(boolean isSuspended)</code> </td>
+</tr>
+<tr id="i2" class="altColor">
+<td class="colFirst"><code>protected boolean</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillBeforeStoreUpdate--">shouldKillBeforeStoreUpdate</a></span>()</code> </td>
+</tr>
+<tr id="i3" class="rowColor">
+<td class="colFirst"><code>protected boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#shouldKillBeforeStoreUpdate-boolean-">shouldKillBeforeStoreUpdate</a></span>(boolean isSuspended)</code> </td>
</tr>
</table>
@@ -213,7 +234,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>killIfSuspended</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.88">killIfSuspended</a></pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.97">killIfSuspended</a></pre>
</li>
</ul>
<a name="killBeforeStoreUpdate">
@@ -222,16 +243,39 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>killBeforeStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.89">killBeforeStoreUpdate</a></pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.103">killBeforeStoreUpdate</a></pre>
+<div class="block">Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is
+ persisting all the state it needs to recover after a crash.</div>
</li>
</ul>
<a name="toggleKillBeforeStoreUpdate">
<!-- -->
</a>
-<ul class="blockListLast">
+<ul class="blockList">
<li class="blockList">
<h4>toggleKillBeforeStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.90">toggleKillBeforeStoreUpdate</a></pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.104">toggleKillBeforeStoreUpdate</a></pre>
+</li>
+</ul>
+<a name="killAfterStoreUpdate">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>killAfterStoreUpdate</h4>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.111">killAfterStoreUpdate</a></pre>
+<div class="block">Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978
+ is about a case where memory-state was being set after store to WAL where a crash could
+ cause us to get stuck. This flag allows killing at what was a vulnerable time.</div>
+</li>
+</ul>
+<a name="toggleKillAfterStoreUpdate">
+<!-- -->
+</a>
+<ul class="blockListLast">
+<li class="blockList">
+<h4>toggleKillAfterStoreUpdate</h4>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.112">toggleKillAfterStoreUpdate</a></pre>
</li>
</ul>
</li>
@@ -248,7 +292,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>Testing</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.87">Testing</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.96">Testing</a>()</pre>
</li>
</ul>
</li>
@@ -265,16 +309,34 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>shouldKillBeforeStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.92">shouldKillBeforeStoreUpdate</a>()</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.114">shouldKillBeforeStoreUpdate</a>()</pre>
</li>
</ul>
<a name="shouldKillBeforeStoreUpdate-boolean-">
<!-- -->
</a>
-<ul class="blockListLast">
+<ul class="blockList">
<li class="blockList">
<h4>shouldKillBeforeStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.101">shouldKillBeforeStoreUpdate</a>(boolean isSuspended)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.123">shouldKillBeforeStoreUpdate</a>(boolean isSuspended)</pre>
+</li>
+</ul>
+<a name="shouldKillAfterStoreUpdate--">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>shouldKillAfterStoreUpdate</h4>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.127">shouldKillAfterStoreUpdate</a>()</pre>
+</li>
+</ul>
+<a name="shouldKillAfterStoreUpdate-boolean-">
+<!-- -->
+</a>
+<ul class="blockListLast">
+<li class="blockList">
+<h4>shouldKillAfterStoreUpdate</h4>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.136">shouldKillAfterStoreUpdate</a>(boolean isSuspended)</pre>
</li>
</ul>
</li>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
index ebe1bde..09af38d 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
@@ -127,7 +127,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1861">ProcedureExecutor.WorkerMonitor</a>
+<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1918">ProcedureExecutor.WorkerMonitor</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.html" title="class in org.apache.hadoop.hbase.procedure2">InlineChore</a></pre>
</li>
</ul>
@@ -277,7 +277,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>WORKER_MONITOR_INTERVAL_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1862">WORKER_MONITOR_INTERVAL_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1919">WORKER_MONITOR_INTERVAL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.WORKER_MONITOR_INTERVAL_CONF_KEY">Constant Field Values</a></dd>
@@ -290,7 +290,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_MONITOR_INTERVAL</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1864">DEFAULT_WORKER_MONITOR_INTERVAL</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1921">DEFAULT_WORKER_MONITOR_INTERVAL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.DEFAULT_WORKER_MONITOR_INTERVAL">Constant Field Values</a></dd>
@@ -303,7 +303,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>WORKER_STUCK_THRESHOLD_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1866">WORKER_STUCK_THRESHOLD_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1923">WORKER_STUCK_THRESHOLD_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.WORKER_STUCK_THRESHOLD_CONF_KEY">Constant Field Values</a></dd>
@@ -316,7 +316,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_STUCK_THRESHOLD</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1868">DEFAULT_WORKER_STUCK_THRESHOLD</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1925">DEFAULT_WORKER_STUCK_THRESHOLD</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.DEFAULT_WORKER_STUCK_THRESHOLD">Constant Field Values</a></dd>
@@ -329,7 +329,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1870">WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1927">WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY">Constant Field Values</a></dd>
@@ -342,7 +342,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_ADD_STUCK_PERCENTAGE</h4>
-<pre>private static final float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1872">DEFAULT_WORKER_ADD_STUCK_PERCENTAGE</a></pre>
+<pre>private static final float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1929">DEFAULT_WORKER_ADD_STUCK_PERCENTAGE</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.DEFAULT_WORKER_ADD_STUCK_PERCENTAGE">Constant Field Values</a></dd>
@@ -355,7 +355,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>addWorkerStuckPercentage</h4>
-<pre>private float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1874">addWorkerStuckPercentage</a></pre>
+<pre>private float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1931">addWorkerStuckPercentage</a></pre>
</li>
</ul>
<a name="timeoutInterval">
@@ -364,7 +364,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>timeoutInterval</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1875">timeoutInterval</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1932">timeoutInterval</a></pre>
</li>
</ul>
<a name="stuckThreshold">
@@ -373,7 +373,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockListLast">
<li class="blockList">
<h4>stuckThreshold</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1876">stuckThreshold</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1933">stuckThreshold</a></pre>
</li>
</ul>
</li>
@@ -390,7 +390,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockListLast">
<li class="blockList">
<h4>WorkerMonitor</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1878">WorkerMonitor</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1935">WorkerMonitor</a>()</pre>
</li>
</ul>
</li>
@@ -407,7 +407,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>run</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1883">run</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1940">run</a>()</pre>
</li>
</ul>
<a name="checkForStuckWorkers--">
@@ -416,7 +416,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>checkForStuckWorkers</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1891">checkForStuckWorkers</a>()</pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1948">checkForStuckWorkers</a>()</pre>
</li>
</ul>
<a name="checkThreadCount-int-">
@@ -425,7 +425,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>checkThreadCount</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1907">checkThreadCount</a>(int stuckCount)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1964">checkThreadCount</a>(int stuckCount)</pre>
</li>
</ul>
<a name="refreshConfig--">
@@ -434,7 +434,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>refreshConfig</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1926">refreshConfig</a>()</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1983">refreshConfig</a>()</pre>
</li>
</ul>
<a name="getTimeoutInterval--">
@@ -443,7 +443,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockListLast">
<li class="blockList">
<h4>getTimeoutInterval</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1936">getTimeoutInterval</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.1993">getTimeoutInterval</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.html#getTimeoutInterval--">getTimeoutInterval</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.html" title="class in org.apache.hadoop.hbase.procedure2">InlineChore</a></code></dd>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
index 4ee7d52..262c812 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
@@ -131,7 +131,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1762">ProcedureExecutor.WorkerThread</a>
+<pre>private class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1819">ProcedureExecutor.WorkerThread</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThread.html" title="class in org.apache.hadoop.hbase.procedure2">StoppableThread</a></pre>
</li>
</ul>
@@ -282,7 +282,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>executionStartTime</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1763">executionStartTime</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1820">executionStartTime</a></pre>
</li>
</ul>
<a name="activeProcedure">
@@ -291,7 +291,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockListLast">
<li class="blockList">
<h4>activeProcedure</h4>
-<pre>private volatile <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1764">activeProcedure</a></pre>
+<pre>private volatile <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1821">activeProcedure</a></pre>
</li>
</ul>
</li>
@@ -308,7 +308,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>WorkerThread</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1766">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1823">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
</li>
</ul>
<a name="WorkerThread-java.lang.ThreadGroup-java.lang.String-">
@@ -317,7 +317,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockListLast">
<li class="blockList">
<h4>WorkerThread</h4>
-<pre>protected <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1770">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group,
+<pre>protected <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1827">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> prefix)</pre>
</li>
</ul>
@@ -335,7 +335,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>sendStopSignal</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1776">sendStopSignal</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1833">sendStopSignal</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThread.html#sendStopSignal--">sendStopSignal</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThread.html" title="class in org.apache.hadoop.hbase.procedure2">StoppableThread</a></code></dd>
@@ -348,7 +348,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>run</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1781">run</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1838">run</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true#run--" title="class or interface in java.lang">run</a></code> in interface <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true" title="class or interface in java.lang">Runnable</a></code></dd>
@@ -363,7 +363,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>toString</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1821">toString</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1878">toString</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true#toString--" title="class or interface in java.lang">toString</a></code> in class <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true" title="class or interface in java.lang">Thread</a></code></dd>
@@ -376,7 +376,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>getCurrentRunTime</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1829">getCurrentRunTime</a>()</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1886">getCurrentRunTime</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the time since the current procedure is running</dd>
@@ -389,7 +389,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockListLast">
<li class="blockList">
<h4>keepAlive</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1834">keepAlive</a>(long lastUpdate)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1891">keepAlive</a>(long lastUpdate)</pre>
</li>
</ul>
</li>
[12/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span c
<TRUNCATED>
[13/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span class="sourceLineNo">
<TRUNCATED>
[18/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
----------------------------------------------------------------------
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
index 3db13c6..0cd57ae 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
@@ -18,7 +18,7 @@
catch(err) {
}
//-->
-var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10,"i38":10,"i39":10,"i40":10,"i41":10,"i42":10,"i43":10,"i44":10,"i45":10,"i46":10,"i47":10,"i48":10,"i49":10,"i50":10,"i51":10,"i52":10,"i53":10,"i54":10,"i55":10,"i56":10,"i57":10,"i58":10,"i59":10,"i60":10,"i61":10,"i62":10,"i63":10,"i64":10};
+var methods = {"i0":10,"i1":10,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10,"i7":10,"i8":10,"i9":10,"i10":10,"i11":10,"i12":10,"i13":10,"i14":10,"i15":10,"i16":10,"i17":10,"i18":10,"i19":10,"i20":10,"i21":10,"i22":10,"i23":10,"i24":10,"i25":10,"i26":10,"i27":10,"i28":10,"i29":10,"i30":10,"i31":10,"i32":10,"i33":10,"i34":10,"i35":10,"i36":10,"i37":10,"i38":10,"i39":10,"i40":10,"i41":10,"i42":10,"i43":10,"i44":10,"i45":10,"i46":10,"i47":10,"i48":10,"i49":10,"i50":10,"i51":10,"i52":10,"i53":10,"i54":10,"i55":10,"i56":10,"i57":10,"i58":10,"i59":10,"i60":10,"i61":10,"i62":10,"i63":10,"i64":10,"i65":10};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
@@ -165,7 +165,9 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static class </code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></span></code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></span></code>
+<div class="block">Class with parameters describing how to fail/die when in testing-context.</div>
+</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>private class </code></td>
@@ -286,7 +288,9 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr class="rowColor">
<td class="colFirst"><code>(package private) <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a></code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing">testing</a></span></code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing">testing</a></span></code>
+<div class="block"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing"><code>testing</code></a> is non-null when ProcedureExecutor is being tested.</div>
+</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a></code></td>
@@ -534,87 +538,91 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
</tr>
<tr id="i35" class="rowColor">
<td class="colFirst"><code>private void</code></td>
-<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#load-boolean-">load</a></span>(boolean abortOnCorruption)</code> </td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#kill-java.lang.String-">kill</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> msg)</code> </td>
</tr>
<tr id="i36" class="altColor">
<td class="colFirst"><code>private void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#load-boolean-">load</a></span>(boolean abortOnCorruption)</code> </td>
+</tr>
+<tr id="i37" class="rowColor">
+<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#loadProcedures-org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator-boolean-">loadProcedures</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a> procIter,
boolean abortOnCorruption)</code> </td>
</tr>
-<tr id="i37" class="rowColor">
+<tr id="i38" class="altColor">
<td class="colFirst"><code>private long</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#nextProcId--">nextProcId</a></span>()</code> </td>
</tr>
-<tr id="i38" class="altColor">
+<tr id="i39" class="rowColor">
<td class="colFirst"><code>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#prepareProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">prepareProcedure</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</code> </td>
</tr>
-<tr id="i39" class="rowColor">
+<tr id="i40" class="altColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#procedureFinished-org.apache.hadoop.hbase.procedure2.Procedure-">procedureFinished</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</code> </td>
</tr>
-<tr id="i40" class="altColor">
+<tr id="i41" class="rowColor">
<td class="colFirst"><code>private long</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#pushProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">pushProcedure</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</code> </td>
</tr>
-<tr id="i41" class="rowColor">
+<tr id="i42" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#refreshConfiguration-org.apache.hadoop.conf.Configuration-">refreshConfiguration</a></span>(org.apache.hadoop.conf.Configuration conf)</code> </td>
</tr>
-<tr id="i42" class="altColor">
+<tr id="i43" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#registerListener-org.apache.hadoop.hbase.procedure2.ProcedureExecutor.ProcedureExecutorListener-">registerListener</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> listener)</code> </td>
</tr>
-<tr id="i43" class="rowColor">
+<tr id="i44" class="altColor">
<td class="colFirst"><code>long</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#registerNonce-org.apache.hadoop.hbase.util.NonceKey-">registerNonce</a></span>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</code>
<div class="block">Register a nonce for a procedure that is going to be submitted.</div>
</td>
</tr>
-<tr id="i44" class="altColor">
+<tr id="i45" class="rowColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#releaseLock-org.apache.hadoop.hbase.procedure2.Procedure-boolean-">releaseLock</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
boolean force)</code> </td>
</tr>
-<tr id="i45" class="rowColor">
+<tr id="i46" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#removeChore-org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore-">removeChore</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</code>
<div class="block">Remove a chore procedure from the executor</div>
</td>
</tr>
-<tr id="i46" class="altColor">
+<tr id="i47" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#removeResult-long-">removeResult</a></span>(long procId)</code>
<div class="block">Mark the specified completed procedure, as ready to remove.</div>
</td>
</tr>
-<tr id="i47" class="rowColor">
+<tr id="i48" class="altColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#restoreLock-org.apache.hadoop.hbase.procedure2.Procedure-java.util.Set-">restoreLock</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> restored)</code> </td>
</tr>
-<tr id="i48" class="altColor">
+<tr id="i49" class="rowColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#restoreLocks--">restoreLocks</a></span>()</code> </td>
</tr>
-<tr id="i49" class="rowColor">
+<tr id="i50" class="altColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#restoreLocks-java.util.Deque-java.util.Set-">restoreLocks</a></span>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> stack,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> restored)</code> </td>
</tr>
-<tr id="i50" class="altColor">
+<tr id="i51" class="rowColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#sendProcedureAddedNotification-long-">sendProcedureAddedNotification</a></span>(long procId)</code> </td>
</tr>
-<tr id="i51" class="rowColor">
+<tr id="i52" class="altColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#sendProcedureFinishedNotification-long-">sendProcedureFinishedNotification</a></span>(long procId)</code> </td>
</tr>
-<tr id="i52" class="altColor">
+<tr id="i53" class="rowColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#sendProcedureLoadedNotification-long-">sendProcedureLoadedNotification</a></span>(long procId)</code> </td>
</tr>
-<tr id="i53" class="rowColor">
+<tr id="i54" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#setFailureResultForNonce-org.apache.hadoop.hbase.util.NonceKey-java.lang.String-org.apache.hadoop.hbase.security.User-java.io.IOException-">setFailureResultForNonce</a></span>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> procName,
@@ -624,61 +632,61 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
same error to the requests with the same nonceKey.</div>
</td>
</tr>
-<tr id="i54" class="altColor">
+<tr id="i55" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#setKeepAliveTime-long-java.util.concurrent.TimeUnit-">setKeepAliveTime</a></span>(long keepAliveTime,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</code> </td>
</tr>
-<tr id="i55" class="rowColor">
+<tr id="i56" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#startWorkers--">startWorkers</a></span>()</code>
<div class="block">Start the workers.</div>
</td>
</tr>
-<tr id="i56" class="altColor">
+<tr id="i57" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#stop--">stop</a></span>()</code> </td>
</tr>
-<tr id="i57" class="rowColor">
+<tr id="i58" class="altColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#submitChildrenProcedures-org.apache.hadoop.hbase.procedure2.Procedure:A-">submitChildrenProcedures</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] subprocs)</code> </td>
</tr>
-<tr id="i58" class="altColor">
+<tr id="i59" class="rowColor">
<td class="colFirst"><code>long</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#submitProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">submitProcedure</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</code>
<div class="block">Add a new root-procedure to the executor.</div>
</td>
</tr>
-<tr id="i59" class="rowColor">
+<tr id="i60" class="altColor">
<td class="colFirst"><code>long</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#submitProcedure-org.apache.hadoop.hbase.procedure2.Procedure-org.apache.hadoop.hbase.util.NonceKey-">submitProcedure</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</code>
<div class="block">Add a new root-procedure to the executor.</div>
</td>
</tr>
-<tr id="i60" class="altColor">
+<tr id="i61" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#submitProcedures-org.apache.hadoop.hbase.procedure2.Procedure:A-">submitProcedures</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] procs)</code>
<div class="block">Add a set of new root-procedure to the executor.</div>
</td>
</tr>
-<tr id="i61" class="rowColor">
+<tr id="i62" class="altColor">
<td class="colFirst"><code>boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#unregisterListener-org.apache.hadoop.hbase.procedure2.ProcedureExecutor.ProcedureExecutorListener-">unregisterListener</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> listener)</code> </td>
</tr>
-<tr id="i62" class="altColor">
+<tr id="i63" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#unregisterNonceIfProcedureWasNotSubmitted-org.apache.hadoop.hbase.util.NonceKey-">unregisterNonceIfProcedureWasNotSubmitted</a></span>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</code>
<div class="block">Remove the NonceKey if the procedure was not submitted to the executor.</div>
</td>
</tr>
-<tr id="i63" class="rowColor">
+<tr id="i64" class="altColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#updateStoreOnExec-org.apache.hadoop.hbase.procedure2.RootProcedureState-org.apache.hadoop.hbase.procedure2.Procedure-org.apache.hadoop.hbase.procedure2.Procedure:A-">updateStoreOnExec</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procedure,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] subprocs)</code> </td>
</tr>
-<tr id="i64" class="altColor">
+<tr id="i65" class="rowColor">
<td class="colFirst"><code>private void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#yieldProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">yieldProcedure</a></span>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</code> </td>
</tr>
@@ -767,7 +775,10 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testing</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.86">testing</a></pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.91">testing</a></pre>
+<div class="block"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing"><code>testing</code></a> is non-null when ProcedureExecutor is being tested. Tests will try to
+ break PE having it fail at various junctures. When non-null, testing is set to an instance of
+ the below internal <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2"><code>ProcedureExecutor.Testing</code></a> class with flags set for the particular test.</div>
</li>
</ul>
<a name="completed">
@@ -776,7 +787,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>completed</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.CompletedProcedureRetainer</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.245">completed</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.CompletedProcedureRetainer</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.280">completed</a></pre>
<div class="block">Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.
Once a Root-Procedure completes (success or failure), the result will be added to this map.
The user of ProcedureExecutor should call getResult(procId) to get the result.</div>
@@ -788,7 +799,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>rollbackStack</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.253">rollbackStack</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.288">rollbackStack</a></pre>
<div class="block">Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.
The RootProcedureState contains the execution stack of the Root-Procedure,
It is added to the map by submitProcedure() and removed on procedure completion.</div>
@@ -800,7 +811,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>procedures</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.260">procedures</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.295">procedures</a></pre>
<div class="block">Helper map to lookup the live procedures by ID.
This map contains every procedure. root-procedures and subprocedures.</div>
</li>
@@ -811,7 +822,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>nonceKeysToProcIdsMap</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.267">nonceKeysToProcIdsMap</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.302">nonceKeysToProcIdsMap</a></pre>
<div class="block">Helper map to lookup whether the procedure already issued from the same client. This map
contains every root procedure.</div>
</li>
@@ -822,7 +833,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>listeners</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.269">listeners</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.304">listeners</a></pre>
</li>
</ul>
<a name="conf">
@@ -831,7 +842,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>conf</h4>
-<pre>private org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.272">conf</a></pre>
+<pre>private org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.307">conf</a></pre>
</li>
</ul>
<a name="threadGroup">
@@ -840,7 +851,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>threadGroup</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.280">threadGroup</a></pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.315">threadGroup</a></pre>
<div class="block">Created in the <code>#start(int, boolean)</code> method. Destroyed in <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#join--"><code>join()</code></a> (FIX! Doing
resource handling rather than observing in a #join is unexpected).
Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery
@@ -853,7 +864,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>workerThreads</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.288">workerThreads</a></pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.323">workerThreads</a></pre>
<div class="block">Created in the <code>#start(int, boolean)</code> method. Terminated in <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#join--"><code>join()</code></a> (FIX! Doing
resource handling rather than observing in a #join is unexpected).
Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery
@@ -866,7 +877,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>timeoutExecutor</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.html" title="class in org.apache.hadoop.hbase.procedure2">TimeoutExecutorThread</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.296">timeoutExecutor</a></pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.html" title="class in org.apache.hadoop.hbase.procedure2">TimeoutExecutorThread</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.331">timeoutExecutor</a></pre>
<div class="block">Created in the <code>#start(int, boolean)</code> method. Terminated in <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#join--"><code>join()</code></a> (FIX! Doing
resource handling rather than observing in a #join is unexpected).
Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery
@@ -879,7 +890,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>corePoolSize</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.298">corePoolSize</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.333">corePoolSize</a></pre>
</li>
</ul>
<a name="maxPoolSize">
@@ -888,7 +899,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>maxPoolSize</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.299">maxPoolSize</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.334">maxPoolSize</a></pre>
</li>
</ul>
<a name="keepAliveTime">
@@ -897,7 +908,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>keepAliveTime</h4>
-<pre>private volatile long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.301">keepAliveTime</a></pre>
+<pre>private volatile long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.336">keepAliveTime</a></pre>
</li>
</ul>
<a name="scheduler">
@@ -906,7 +917,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>scheduler</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.306">scheduler</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.341">scheduler</a></pre>
<div class="block">Scheduler/Queue that contains runnable procedures.</div>
</li>
</ul>
@@ -916,7 +927,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>lastProcId</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.308">lastProcId</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.343">lastProcId</a></pre>
</li>
</ul>
<a name="workerId">
@@ -925,7 +936,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>workerId</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.309">workerId</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.344">workerId</a></pre>
</li>
</ul>
<a name="activeExecutorCount">
@@ -934,7 +945,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>activeExecutorCount</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.310">activeExecutorCount</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.345">activeExecutorCount</a></pre>
</li>
</ul>
<a name="running">
@@ -943,7 +954,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>running</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicBoolean.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicBoolean</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.311">running</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicBoolean.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicBoolean</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.346">running</a></pre>
</li>
</ul>
<a name="environment">
@@ -952,7 +963,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>environment</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.312">environment</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.347">environment</a></pre>
</li>
</ul>
<a name="store">
@@ -961,7 +972,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>store</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.313">store</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.348">store</a></pre>
</li>
</ul>
<a name="checkOwnerSet">
@@ -970,7 +981,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>checkOwnerSet</h4>
-<pre>private final boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.315">checkOwnerSet</a></pre>
+<pre>private final boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.350">checkOwnerSet</a></pre>
</li>
</ul>
<a name="procExecutionLock">
@@ -979,7 +990,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>procExecutionLock</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/util/IdLock.html" title="class in org.apache.hadoop.hbase.util">IdLock</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.323">procExecutionLock</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/util/IdLock.html" title="class in org.apache.hadoop.hbase.util">IdLock</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.358">procExecutionLock</a></pre>
</li>
</ul>
</li>
@@ -998,7 +1009,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>ProcedureExecutor</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.325">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.360">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> environment,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> store)</pre>
</li>
@@ -1011,7 +1022,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>ProcedureExecutor</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.330">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.365">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> environment,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> store,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> scheduler)</pre>
@@ -1031,7 +1042,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>load</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.341">load</a>(boolean abortOnCorruption)
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.376">load</a>(boolean abortOnCorruption)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1045,7 +1056,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>restoreLock</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.374">restoreLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.409">restoreLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> restored)</pre>
</li>
</ul>
@@ -1055,7 +1066,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>restoreLocks</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.379">restoreLocks</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> stack,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.414">restoreLocks</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> stack,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> restored)</pre>
</li>
</ul>
@@ -1065,7 +1076,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>restoreLocks</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.398">restoreLocks</a>()</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.433">restoreLocks</a>()</pre>
</li>
</ul>
<a name="loadProcedures-org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator-boolean-">
@@ -1074,7 +1085,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>loadProcedures</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.418">loadProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a> procIter,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.453">loadProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a> procIter,
boolean abortOnCorruption)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -1089,7 +1100,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>init</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.581">init</a>(int numThreads,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.627">init</a>(int numThreads,
boolean abortOnCorruption)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Initialize the procedure executor, but do not start workers. We will start them later.
@@ -1113,7 +1124,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>startWorkers</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.626">startWorkers</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.672">startWorkers</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Start the workers.</div>
<dl>
@@ -1128,7 +1139,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>stop</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.645">stop</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.691">stop</a>()</pre>
</li>
</ul>
<a name="join--">
@@ -1137,7 +1148,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>join</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.656">join</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.702">join</a>()</pre>
</li>
</ul>
<a name="refreshConfiguration-org.apache.hadoop.conf.Configuration-">
@@ -1146,7 +1157,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>refreshConfiguration</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.687">refreshConfiguration</a>(org.apache.hadoop.conf.Configuration conf)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.733">refreshConfiguration</a>(org.apache.hadoop.conf.Configuration conf)</pre>
</li>
</ul>
<a name="isRunning--">
@@ -1155,7 +1166,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isRunning</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.696">isRunning</a>()</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.742">isRunning</a>()</pre>
</li>
</ul>
<a name="getWorkerThreadCount--">
@@ -1164,7 +1175,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getWorkerThreadCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.703">getWorkerThreadCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.749">getWorkerThreadCount</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the current number of worker threads.</dd>
@@ -1177,7 +1188,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getCorePoolSize</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.710">getCorePoolSize</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.756">getCorePoolSize</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the core pool size settings.</dd>
@@ -1190,7 +1201,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getActiveExecutorCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.714">getActiveExecutorCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.760">getActiveExecutorCount</a>()</pre>
</li>
</ul>
<a name="getEnvironment--">
@@ -1199,7 +1210,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getEnvironment</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.718">getEnvironment</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.764">getEnvironment</a>()</pre>
</li>
</ul>
<a name="getStore--">
@@ -1208,7 +1219,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getStore</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.722">getStore</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.768">getStore</a>()</pre>
</li>
</ul>
<a name="getScheduler--">
@@ -1217,7 +1228,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getScheduler</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.726">getScheduler</a>()</pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.772">getScheduler</a>()</pre>
</li>
</ul>
<a name="setKeepAliveTime-long-java.util.concurrent.TimeUnit-">
@@ -1226,7 +1237,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setKeepAliveTime</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.730">setKeepAliveTime</a>(long keepAliveTime,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.776">setKeepAliveTime</a>(long keepAliveTime,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</pre>
</li>
</ul>
@@ -1236,7 +1247,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getKeepAliveTime</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.735">getKeepAliveTime</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.781">getKeepAliveTime</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</pre>
</li>
</ul>
<a name="addChore-org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore-">
@@ -1245,7 +1256,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>addChore</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.747">addChore</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.793">addChore</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
<div class="block">Add a chore procedure to the executor</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1259,7 +1270,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>removeChore</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.757">removeChore</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.803">removeChore</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
<div class="block">Remove a chore procedure from the executor</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1275,7 +1286,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>createNonceKey</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.771">createNonceKey</a>(long nonceGroup,
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.817">createNonceKey</a>(long nonceGroup,
long nonce)</pre>
<div class="block">Create a NoneKey from the specified nonceGroup and nonce.</div>
<dl>
@@ -1293,7 +1304,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>registerNonce</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.786">registerNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.832">registerNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
<div class="block">Register a nonce for a procedure that is going to be submitted.
A procId will be reserved and on submitProcedure(),
the procedure with the specified nonce will take the reserved ProcId.
@@ -1314,7 +1325,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>unregisterNonceIfProcedureWasNotSubmitted</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.817">unregisterNonceIfProcedureWasNotSubmitted</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.863">unregisterNonceIfProcedureWasNotSubmitted</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
<div class="block">Remove the NonceKey if the procedure was not submitted to the executor.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1328,7 +1339,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setFailureResultForNonce</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.891">setFailureResultForNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.937">setFailureResultForNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> procName,
<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a> procOwner,
<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a> exception)</pre>
@@ -1349,7 +1360,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedure</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.916">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.962">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
<div class="block">Add a new root-procedure to the executor.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1365,7 +1376,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedure</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.928">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.974">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
<div class="block">Add a new root-procedure to the executor.</div>
<dl>
@@ -1383,7 +1394,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedures</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.959">submitProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] procs)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1005">submitProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] procs)</pre>
<div class="block">Add a set of new root-procedure to the executor.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1397,7 +1408,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>prepareProcedure</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.982">prepareProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1028">prepareProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="pushProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1406,7 +1417,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>pushProcedure</h4>
-<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.991">pushProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1037">pushProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="abort-long-">
@@ -1415,7 +1426,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>abort</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1015">abort</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1061">abort</a>(long procId)</pre>
<div class="block">Send an abort notification the specified procedure.
Depending on the procedure implementation the abort can be considered or ignored.</div>
<dl>
@@ -1432,7 +1443,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>abort</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1026">abort</a>(long procId,
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1072">abort</a>(long procId,
boolean mayInterruptIfRunning)</pre>
<div class="block">Send an abort notification to the specified procedure.
Depending on the procedure implementation, the abort can be considered or ignored.</div>
@@ -1451,7 +1462,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedure</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1040">getProcedure</a>(long procId)</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1086">getProcedure</a>(long procId)</pre>
</li>
</ul>
<a name="getProcedure-java.lang.Class-long-">
@@ -1460,7 +1471,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedure</h4>
-<pre>public <T extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> T <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1044">getProcedure</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Class.html?is-external=true" title="class or interface in java.lang">Class</a><T> clazz,
+<pre>public <T extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> T <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1090">getProcedure</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Class.html?is-external=true" title="class or interface in java.lang">Class</a><T> clazz,
long procId)</pre>
</li>
</ul>
@@ -1470,7 +1481,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getResult</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1052">getResult</a>(long procId)</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1098">getResult</a>(long procId)</pre>
</li>
</ul>
<a name="isFinished-long-">
@@ -1479,7 +1490,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isFinished</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1068">isFinished</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1114">isFinished</a>(long procId)</pre>
<div class="block">Return true if the procedure is finished.
The state may be "completed successfully" or "failed and rolledback".
Use getResult() to check the state or get the result data.</div>
@@ -1497,7 +1508,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isStarted</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1077">isStarted</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1123">isStarted</a>(long procId)</pre>
<div class="block">Return true if the procedure is started.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1513,7 +1524,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>removeResult</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1089">removeResult</a>(long procId)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1135">removeResult</a>(long procId)</pre>
<div class="block">Mark the specified completed procedure, as ready to remove.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1527,7 +1538,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getResultOrProcedure</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1101">getResultOrProcedure</a>(long procId)</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1147">getResultOrProcedure</a>(long procId)</pre>
</li>
</ul>
<a name="isProcedureOwner-long-org.apache.hadoop.hbase.security.User-">
@@ -1536,7 +1547,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isProcedureOwner</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1117">isProcedureOwner</a>(long procId,
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1163">isProcedureOwner</a>(long procId,
<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a> user)</pre>
<div class="block">Check if the user is this procedure's owner</div>
<dl>
@@ -1555,7 +1566,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedures</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1140">getProcedures</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1186">getProcedures</a>()</pre>
<div class="block">Get procedures.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1569,7 +1580,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/jav
<TRUNCATED>
[08/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
index 0a656db..e8799af 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
@@ -91,1861 +91,1918 @@
<span class="sourceLineNo">083</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.83"></a>
<span class="sourceLineNo">084</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.84"></a>
<span class="sourceLineNo">085</span><a name="line.85"></a>
-<span class="sourceLineNo">086</span> Testing testing = null;<a name="line.86"></a>
-<span class="sourceLineNo">087</span> public static class Testing {<a name="line.87"></a>
-<span class="sourceLineNo">088</span> protected boolean killIfSuspended = false;<a name="line.88"></a>
-<span class="sourceLineNo">089</span> protected boolean killBeforeStoreUpdate = false;<a name="line.89"></a>
-<span class="sourceLineNo">090</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.90"></a>
-<span class="sourceLineNo">091</span><a name="line.91"></a>
-<span class="sourceLineNo">092</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.92"></a>
-<span class="sourceLineNo">093</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.93"></a>
-<span class="sourceLineNo">094</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.94"></a>
-<span class="sourceLineNo">095</span> this.killBeforeStoreUpdate = !kill;<a name="line.95"></a>
-<span class="sourceLineNo">096</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.96"></a>
-<span class="sourceLineNo">097</span> }<a name="line.97"></a>
-<span class="sourceLineNo">098</span> return kill;<a name="line.98"></a>
-<span class="sourceLineNo">099</span> }<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.101"></a>
-<span class="sourceLineNo">102</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.102"></a>
-<span class="sourceLineNo">103</span> }<a name="line.103"></a>
-<span class="sourceLineNo">104</span> }<a name="line.104"></a>
+<span class="sourceLineNo">086</span> /**<a name="line.86"></a>
+<span class="sourceLineNo">087</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.87"></a>
+<span class="sourceLineNo">088</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.88"></a>
+<span class="sourceLineNo">089</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.89"></a>
+<span class="sourceLineNo">090</span> */<a name="line.90"></a>
+<span class="sourceLineNo">091</span> Testing testing = null;<a name="line.91"></a>
+<span class="sourceLineNo">092</span><a name="line.92"></a>
+<span class="sourceLineNo">093</span> /**<a name="line.93"></a>
+<span class="sourceLineNo">094</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.94"></a>
+<span class="sourceLineNo">095</span> */<a name="line.95"></a>
+<span class="sourceLineNo">096</span> public static class Testing {<a name="line.96"></a>
+<span class="sourceLineNo">097</span> protected boolean killIfSuspended = false;<a name="line.97"></a>
+<span class="sourceLineNo">098</span><a name="line.98"></a>
+<span class="sourceLineNo">099</span> /**<a name="line.99"></a>
+<span class="sourceLineNo">100</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.100"></a>
+<span class="sourceLineNo">101</span> * persisting all the state it needs to recover after a crash.<a name="line.101"></a>
+<span class="sourceLineNo">102</span> */<a name="line.102"></a>
+<span class="sourceLineNo">103</span> protected boolean killBeforeStoreUpdate = false;<a name="line.103"></a>
+<span class="sourceLineNo">104</span> protected boolean toggleKillBeforeStoreUpdate = false;<a name="line.104"></a>
<span class="sourceLineNo">105</span><a name="line.105"></a>
-<span class="sourceLineNo">106</span> public interface ProcedureExecutorListener {<a name="line.106"></a>
-<span class="sourceLineNo">107</span> void procedureLoaded(long procId);<a name="line.107"></a>
-<span class="sourceLineNo">108</span> void procedureAdded(long procId);<a name="line.108"></a>
-<span class="sourceLineNo">109</span> void procedureFinished(long procId);<a name="line.109"></a>
-<span class="sourceLineNo">110</span> }<a name="line.110"></a>
-<span class="sourceLineNo">111</span><a name="line.111"></a>
-<span class="sourceLineNo">112</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.112"></a>
-<span class="sourceLineNo">113</span> private final Procedure<TEnvironment> procedure;<a name="line.113"></a>
-<span class="sourceLineNo">114</span> private long clientAckTime;<a name="line.114"></a>
-<span class="sourceLineNo">115</span><a name="line.115"></a>
-<span class="sourceLineNo">116</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.116"></a>
-<span class="sourceLineNo">117</span> this.procedure = procedure;<a name="line.117"></a>
-<span class="sourceLineNo">118</span> clientAckTime = -1;<a name="line.118"></a>
-<span class="sourceLineNo">119</span> }<a name="line.119"></a>
-<span class="sourceLineNo">120</span><a name="line.120"></a>
-<span class="sourceLineNo">121</span> public Procedure<TEnvironment> getProcedure() {<a name="line.121"></a>
-<span class="sourceLineNo">122</span> return procedure;<a name="line.122"></a>
-<span class="sourceLineNo">123</span> }<a name="line.123"></a>
-<span class="sourceLineNo">124</span><a name="line.124"></a>
-<span class="sourceLineNo">125</span> public boolean hasClientAckTime() {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> return clientAckTime != -1;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> }<a name="line.127"></a>
-<span class="sourceLineNo">128</span><a name="line.128"></a>
-<span class="sourceLineNo">129</span> public long getClientAckTime() {<a name="line.129"></a>
-<span class="sourceLineNo">130</span> return clientAckTime;<a name="line.130"></a>
-<span class="sourceLineNo">131</span> }<a name="line.131"></a>
-<span class="sourceLineNo">132</span><a name="line.132"></a>
-<span class="sourceLineNo">133</span> public void setClientAckTime(long clientAckTime) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> this.clientAckTime = clientAckTime;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span><a name="line.136"></a>
-<span class="sourceLineNo">137</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.137"></a>
-<span class="sourceLineNo">138</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.138"></a>
-<span class="sourceLineNo">139</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span> }<a name="line.141"></a>
-<span class="sourceLineNo">142</span><a name="line.142"></a>
-<span class="sourceLineNo">143</span> /**<a name="line.143"></a>
-<span class="sourceLineNo">144</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.144"></a>
-<span class="sourceLineNo">145</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.145"></a>
-<span class="sourceLineNo">146</span> *<a name="line.146"></a>
-<span class="sourceLineNo">147</span> * <p>Since the client code looks more or less like:<a name="line.147"></a>
-<span class="sourceLineNo">148</span> * <pre><a name="line.148"></a>
-<span class="sourceLineNo">149</span> * procId = master.doOperation()<a name="line.149"></a>
-<span class="sourceLineNo">150</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.150"></a>
-<span class="sourceLineNo">151</span> * </pre><a name="line.151"></a>
-<span class="sourceLineNo">152</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.152"></a>
-<span class="sourceLineNo">153</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.153"></a>
-<span class="sourceLineNo">154</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.154"></a>
-<span class="sourceLineNo">155</span> * which will return the result/state to the client, and it will mark the completed<a name="line.155"></a>
-<span class="sourceLineNo">156</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.156"></a>
-<span class="sourceLineNo">157</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.157"></a>
-<span class="sourceLineNo">158</span> * the proc result the client will be able to get the result the next try.<a name="line.158"></a>
-<span class="sourceLineNo">159</span> */<a name="line.159"></a>
-<span class="sourceLineNo">160</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.160"></a>
-<span class="sourceLineNo">161</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.161"></a>
-<span class="sourceLineNo">162</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.162"></a>
+<span class="sourceLineNo">106</span> /**<a name="line.106"></a>
+<span class="sourceLineNo">107</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.107"></a>
+<span class="sourceLineNo">108</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.108"></a>
+<span class="sourceLineNo">109</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.109"></a>
+<span class="sourceLineNo">110</span> */<a name="line.110"></a>
+<span class="sourceLineNo">111</span> protected boolean killAfterStoreUpdate = false;<a name="line.111"></a>
+<span class="sourceLineNo">112</span> protected boolean toggleKillAfterStoreUpdate = false;<a name="line.112"></a>
+<span class="sourceLineNo">113</span><a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.114"></a>
+<span class="sourceLineNo">115</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.115"></a>
+<span class="sourceLineNo">116</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.116"></a>
+<span class="sourceLineNo">117</span> this.killBeforeStoreUpdate = !kill;<a name="line.117"></a>
+<span class="sourceLineNo">118</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.118"></a>
+<span class="sourceLineNo">119</span> }<a name="line.119"></a>
+<span class="sourceLineNo">120</span> return kill;<a name="line.120"></a>
+<span class="sourceLineNo">121</span> }<a name="line.121"></a>
+<span class="sourceLineNo">122</span><a name="line.122"></a>
+<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate(final boolean isSuspended) {<a name="line.123"></a>
+<span class="sourceLineNo">124</span> return (isSuspended && !killIfSuspended) ? false : shouldKillBeforeStoreUpdate();<a name="line.124"></a>
+<span class="sourceLineNo">125</span> }<a name="line.125"></a>
+<span class="sourceLineNo">126</span><a name="line.126"></a>
+<span class="sourceLineNo">127</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.127"></a>
+<span class="sourceLineNo">128</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.128"></a>
+<span class="sourceLineNo">129</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.129"></a>
+<span class="sourceLineNo">130</span> this.killAfterStoreUpdate = !kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.131"></a>
+<span class="sourceLineNo">132</span> }<a name="line.132"></a>
+<span class="sourceLineNo">133</span> return kill;<a name="line.133"></a>
+<span class="sourceLineNo">134</span> }<a name="line.134"></a>
+<span class="sourceLineNo">135</span><a name="line.135"></a>
+<span class="sourceLineNo">136</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.136"></a>
+<span class="sourceLineNo">137</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.137"></a>
+<span class="sourceLineNo">138</span> }<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span><a name="line.140"></a>
+<span class="sourceLineNo">141</span> public interface ProcedureExecutorListener {<a name="line.141"></a>
+<span class="sourceLineNo">142</span> void procedureLoaded(long procId);<a name="line.142"></a>
+<span class="sourceLineNo">143</span> void procedureAdded(long procId);<a name="line.143"></a>
+<span class="sourceLineNo">144</span> void procedureFinished(long procId);<a name="line.144"></a>
+<span class="sourceLineNo">145</span> }<a name="line.145"></a>
+<span class="sourceLineNo">146</span><a name="line.146"></a>
+<span class="sourceLineNo">147</span> private static final class CompletedProcedureRetainer<TEnvironment> {<a name="line.147"></a>
+<span class="sourceLineNo">148</span> private final Procedure<TEnvironment> procedure;<a name="line.148"></a>
+<span class="sourceLineNo">149</span> private long clientAckTime;<a name="line.149"></a>
+<span class="sourceLineNo">150</span><a name="line.150"></a>
+<span class="sourceLineNo">151</span> public CompletedProcedureRetainer(Procedure<TEnvironment> procedure) {<a name="line.151"></a>
+<span class="sourceLineNo">152</span> this.procedure = procedure;<a name="line.152"></a>
+<span class="sourceLineNo">153</span> clientAckTime = -1;<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span><a name="line.155"></a>
+<span class="sourceLineNo">156</span> public Procedure<TEnvironment> getProcedure() {<a name="line.156"></a>
+<span class="sourceLineNo">157</span> return procedure;<a name="line.157"></a>
+<span class="sourceLineNo">158</span> }<a name="line.158"></a>
+<span class="sourceLineNo">159</span><a name="line.159"></a>
+<span class="sourceLineNo">160</span> public boolean hasClientAckTime() {<a name="line.160"></a>
+<span class="sourceLineNo">161</span> return clientAckTime != -1;<a name="line.161"></a>
+<span class="sourceLineNo">162</span> }<a name="line.162"></a>
<span class="sourceLineNo">163</span><a name="line.163"></a>
-<span class="sourceLineNo">164</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.164"></a>
-<span class="sourceLineNo">165</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.165"></a>
-<span class="sourceLineNo">166</span><a name="line.166"></a>
-<span class="sourceLineNo">167</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.167"></a>
-<span class="sourceLineNo">168</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.170"></a>
-<span class="sourceLineNo">171</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.171"></a>
-<span class="sourceLineNo">172</span><a name="line.172"></a>
-<span class="sourceLineNo">173</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.173"></a>
-<span class="sourceLineNo">174</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.174"></a>
-<span class="sourceLineNo">175</span><a name="line.175"></a>
-<span class="sourceLineNo">176</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.176"></a>
-<span class="sourceLineNo">177</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.177"></a>
-<span class="sourceLineNo">178</span> private final ProcedureStore store;<a name="line.178"></a>
-<span class="sourceLineNo">179</span> private Configuration conf;<a name="line.179"></a>
-<span class="sourceLineNo">180</span><a name="line.180"></a>
-<span class="sourceLineNo">181</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.181"></a>
-<span class="sourceLineNo">182</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.182"></a>
-<span class="sourceLineNo">183</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.183"></a>
-<span class="sourceLineNo">184</span> // set the timeout interval that triggers the periodic-procedure<a name="line.184"></a>
-<span class="sourceLineNo">185</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.185"></a>
-<span class="sourceLineNo">186</span> this.completed = completedMap;<a name="line.186"></a>
-<span class="sourceLineNo">187</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.187"></a>
-<span class="sourceLineNo">188</span> this.store = store;<a name="line.188"></a>
-<span class="sourceLineNo">189</span> this.conf = conf;<a name="line.189"></a>
-<span class="sourceLineNo">190</span> }<a name="line.190"></a>
-<span class="sourceLineNo">191</span><a name="line.191"></a>
-<span class="sourceLineNo">192</span> @Override<a name="line.192"></a>
-<span class="sourceLineNo">193</span> protected void periodicExecute(final TEnvironment env) {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> if (completed.isEmpty()) {<a name="line.194"></a>
-<span class="sourceLineNo">195</span> if (LOG.isTraceEnabled()) {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> LOG.trace("No completed procedures to cleanup.");<a name="line.196"></a>
-<span class="sourceLineNo">197</span> }<a name="line.197"></a>
-<span class="sourceLineNo">198</span> return;<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.201"></a>
-<span class="sourceLineNo">202</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.203"></a>
+<span class="sourceLineNo">164</span> public long getClientAckTime() {<a name="line.164"></a>
+<span class="sourceLineNo">165</span> return clientAckTime;<a name="line.165"></a>
+<span class="sourceLineNo">166</span> }<a name="line.166"></a>
+<span class="sourceLineNo">167</span><a name="line.167"></a>
+<span class="sourceLineNo">168</span> public void setClientAckTime(long clientAckTime) {<a name="line.168"></a>
+<span class="sourceLineNo">169</span> this.clientAckTime = clientAckTime;<a name="line.169"></a>
+<span class="sourceLineNo">170</span> }<a name="line.170"></a>
+<span class="sourceLineNo">171</span><a name="line.171"></a>
+<span class="sourceLineNo">172</span> public boolean isExpired(long now, long evictTtl, long evictAckTtl) {<a name="line.172"></a>
+<span class="sourceLineNo">173</span> return (hasClientAckTime() && (now - getClientAckTime()) >= evictAckTtl) ||<a name="line.173"></a>
+<span class="sourceLineNo">174</span> (now - procedure.getLastUpdate()) >= evictTtl;<a name="line.174"></a>
+<span class="sourceLineNo">175</span> }<a name="line.175"></a>
+<span class="sourceLineNo">176</span> }<a name="line.176"></a>
+<span class="sourceLineNo">177</span><a name="line.177"></a>
+<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
+<span class="sourceLineNo">179</span> * Internal cleaner that removes the completed procedure results after a TTL.<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * NOTE: This is a special case handled in timeoutLoop().<a name="line.180"></a>
+<span class="sourceLineNo">181</span> *<a name="line.181"></a>
+<span class="sourceLineNo">182</span> * <p>Since the client code looks more or less like:<a name="line.182"></a>
+<span class="sourceLineNo">183</span> * <pre><a name="line.183"></a>
+<span class="sourceLineNo">184</span> * procId = master.doOperation()<a name="line.184"></a>
+<span class="sourceLineNo">185</span> * while (master.getProcResult(procId) == ProcInProgress);<a name="line.185"></a>
+<span class="sourceLineNo">186</span> * </pre><a name="line.186"></a>
+<span class="sourceLineNo">187</span> * The master should not throw away the proc result as soon as the procedure is done<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * but should wait a result request from the client (see executor.removeResult(procId))<a name="line.188"></a>
+<span class="sourceLineNo">189</span> * The client will call something like master.isProcDone() or master.getProcResult()<a name="line.189"></a>
+<span class="sourceLineNo">190</span> * which will return the result/state to the client, and it will mark the completed<a name="line.190"></a>
+<span class="sourceLineNo">191</span> * proc as ready to delete. note that the client may not receive the response from<a name="line.191"></a>
+<span class="sourceLineNo">192</span> * the master (e.g. master failover) so, if we delay a bit the real deletion of<a name="line.192"></a>
+<span class="sourceLineNo">193</span> * the proc result the client will be able to get the result the next try.<a name="line.193"></a>
+<span class="sourceLineNo">194</span> */<a name="line.194"></a>
+<span class="sourceLineNo">195</span> private static class CompletedProcedureCleaner<TEnvironment><a name="line.195"></a>
+<span class="sourceLineNo">196</span> extends ProcedureInMemoryChore<TEnvironment> {<a name="line.196"></a>
+<span class="sourceLineNo">197</span> private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureCleaner.class);<a name="line.197"></a>
+<span class="sourceLineNo">198</span><a name="line.198"></a>
+<span class="sourceLineNo">199</span> private static final String CLEANER_INTERVAL_CONF_KEY = "hbase.procedure.cleaner.interval";<a name="line.199"></a>
+<span class="sourceLineNo">200</span> private static final int DEFAULT_CLEANER_INTERVAL = 30 * 1000; // 30sec<a name="line.200"></a>
+<span class="sourceLineNo">201</span><a name="line.201"></a>
+<span class="sourceLineNo">202</span> private static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.203"></a>
<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> final long[] batchIds = new long[batchSize];<a name="line.205"></a>
-<span class="sourceLineNo">206</span> int batchCount = 0;<a name="line.206"></a>
+<span class="sourceLineNo">205</span> private static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.205"></a>
+<span class="sourceLineNo">206</span> private static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.206"></a>
<span class="sourceLineNo">207</span><a name="line.207"></a>
-<span class="sourceLineNo">208</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.208"></a>
-<span class="sourceLineNo">209</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.209"></a>
-<span class="sourceLineNo">210</span> completed.entrySet().iterator();<a name="line.210"></a>
-<span class="sourceLineNo">211</span> while (it.hasNext() && store.isRunning()) {<a name="line.211"></a>
-<span class="sourceLineNo">212</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.212"></a>
-<span class="sourceLineNo">213</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.214"></a>
+<span class="sourceLineNo">208</span> private static final String BATCH_SIZE_CONF_KEY = "hbase.procedure.cleaner.evict.batch.size";<a name="line.208"></a>
+<span class="sourceLineNo">209</span> private static final int DEFAULT_BATCH_SIZE = 32;<a name="line.209"></a>
+<span class="sourceLineNo">210</span><a name="line.210"></a>
+<span class="sourceLineNo">211</span> private final Map<Long, CompletedProcedureRetainer<TEnvironment>> completed;<a name="line.211"></a>
+<span class="sourceLineNo">212</span> private final Map<NonceKey, Long> nonceKeysToProcIdsMap;<a name="line.212"></a>
+<span class="sourceLineNo">213</span> private final ProcedureStore store;<a name="line.213"></a>
+<span class="sourceLineNo">214</span> private Configuration conf;<a name="line.214"></a>
<span class="sourceLineNo">215</span><a name="line.215"></a>
-<span class="sourceLineNo">216</span> // TODO: Select TTL based on Procedure type<a name="line.216"></a>
-<span class="sourceLineNo">217</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.217"></a>
-<span class="sourceLineNo">218</span> // Failed procedures aren't persisted in WAL.<a name="line.218"></a>
-<span class="sourceLineNo">219</span> if (!(proc instanceof FailedProcedure)) {<a name="line.219"></a>
-<span class="sourceLineNo">220</span> batchIds[batchCount++] = entry.getKey();<a name="line.220"></a>
-<span class="sourceLineNo">221</span> if (batchCount == batchIds.length) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> store.delete(batchIds, 0, batchCount);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> batchCount = 0;<a name="line.223"></a>
-<span class="sourceLineNo">224</span> }<a name="line.224"></a>
-<span class="sourceLineNo">225</span> }<a name="line.225"></a>
-<span class="sourceLineNo">226</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.226"></a>
-<span class="sourceLineNo">227</span> if (nonceKey != null) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.228"></a>
-<span class="sourceLineNo">229</span> }<a name="line.229"></a>
-<span class="sourceLineNo">230</span> it.remove();<a name="line.230"></a>
-<span class="sourceLineNo">231</span> LOG.trace("Evict completed {}", proc);<a name="line.231"></a>
+<span class="sourceLineNo">216</span> public CompletedProcedureCleaner(Configuration conf, final ProcedureStore store,<a name="line.216"></a>
+<span class="sourceLineNo">217</span> final Map<Long, CompletedProcedureRetainer<TEnvironment>> completedMap,<a name="line.217"></a>
+<span class="sourceLineNo">218</span> final Map<NonceKey, Long> nonceKeysToProcIdsMap) {<a name="line.218"></a>
+<span class="sourceLineNo">219</span> // set the timeout interval that triggers the periodic-procedure<a name="line.219"></a>
+<span class="sourceLineNo">220</span> super(conf.getInt(CLEANER_INTERVAL_CONF_KEY, DEFAULT_CLEANER_INTERVAL));<a name="line.220"></a>
+<span class="sourceLineNo">221</span> this.completed = completedMap;<a name="line.221"></a>
+<span class="sourceLineNo">222</span> this.nonceKeysToProcIdsMap = nonceKeysToProcIdsMap;<a name="line.222"></a>
+<span class="sourceLineNo">223</span> this.store = store;<a name="line.223"></a>
+<span class="sourceLineNo">224</span> this.conf = conf;<a name="line.224"></a>
+<span class="sourceLineNo">225</span> }<a name="line.225"></a>
+<span class="sourceLineNo">226</span><a name="line.226"></a>
+<span class="sourceLineNo">227</span> @Override<a name="line.227"></a>
+<span class="sourceLineNo">228</span> protected void periodicExecute(final TEnvironment env) {<a name="line.228"></a>
+<span class="sourceLineNo">229</span> if (completed.isEmpty()) {<a name="line.229"></a>
+<span class="sourceLineNo">230</span> if (LOG.isTraceEnabled()) {<a name="line.230"></a>
+<span class="sourceLineNo">231</span> LOG.trace("No completed procedures to cleanup.");<a name="line.231"></a>
<span class="sourceLineNo">232</span> }<a name="line.232"></a>
-<span class="sourceLineNo">233</span> }<a name="line.233"></a>
-<span class="sourceLineNo">234</span> if (batchCount > 0) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> store.delete(batchIds, 0, batchCount);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">233</span> return;<a name="line.233"></a>
+<span class="sourceLineNo">234</span> }<a name="line.234"></a>
+<span class="sourceLineNo">235</span><a name="line.235"></a>
+<span class="sourceLineNo">236</span> final long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> final long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.237"></a>
+<span class="sourceLineNo">238</span> final int batchSize = conf.getInt(BATCH_SIZE_CONF_KEY, DEFAULT_BATCH_SIZE);<a name="line.238"></a>
<span class="sourceLineNo">239</span><a name="line.239"></a>
-<span class="sourceLineNo">240</span> /**<a name="line.240"></a>
-<span class="sourceLineNo">241</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.241"></a>
-<span class="sourceLineNo">242</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.242"></a>
-<span class="sourceLineNo">243</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.243"></a>
-<span class="sourceLineNo">244</span> */<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.245"></a>
-<span class="sourceLineNo">246</span> new ConcurrentHashMap<>();<a name="line.246"></a>
-<span class="sourceLineNo">247</span><a name="line.247"></a>
-<span class="sourceLineNo">248</span> /**<a name="line.248"></a>
-<span class="sourceLineNo">249</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.249"></a>
-<span class="sourceLineNo">250</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.250"></a>
-<span class="sourceLineNo">251</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> */<a name="line.252"></a>
-<span class="sourceLineNo">253</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.253"></a>
-<span class="sourceLineNo">254</span> new ConcurrentHashMap<>();<a name="line.254"></a>
-<span class="sourceLineNo">255</span><a name="line.255"></a>
-<span class="sourceLineNo">256</span> /**<a name="line.256"></a>
-<span class="sourceLineNo">257</span> * Helper map to lookup the live procedures by ID.<a name="line.257"></a>
-<span class="sourceLineNo">258</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.258"></a>
-<span class="sourceLineNo">259</span> */<a name="line.259"></a>
-<span class="sourceLineNo">260</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.260"></a>
-<span class="sourceLineNo">261</span> new ConcurrentHashMap<>();<a name="line.261"></a>
-<span class="sourceLineNo">262</span><a name="line.262"></a>
-<span class="sourceLineNo">263</span> /**<a name="line.263"></a>
-<span class="sourceLineNo">264</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.264"></a>
-<span class="sourceLineNo">265</span> * contains every root procedure.<a name="line.265"></a>
-<span class="sourceLineNo">266</span> */<a name="line.266"></a>
-<span class="sourceLineNo">267</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.269"></a>
-<span class="sourceLineNo">270</span> new CopyOnWriteArrayList<>();<a name="line.270"></a>
-<span class="sourceLineNo">271</span><a name="line.271"></a>
-<span class="sourceLineNo">272</span> private Configuration conf;<a name="line.272"></a>
-<span class="sourceLineNo">273</span><a name="line.273"></a>
-<span class="sourceLineNo">274</span> /**<a name="line.274"></a>
-<span class="sourceLineNo">275</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.275"></a>
-<span class="sourceLineNo">276</span> * resource handling rather than observing in a #join is unexpected).<a name="line.276"></a>
-<span class="sourceLineNo">277</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.277"></a>
-<span class="sourceLineNo">278</span> * (Should be ok).<a name="line.278"></a>
+<span class="sourceLineNo">240</span> final long[] batchIds = new long[batchSize];<a name="line.240"></a>
+<span class="sourceLineNo">241</span> int batchCount = 0;<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> final long now = EnvironmentEdgeManager.currentTime();<a name="line.243"></a>
+<span class="sourceLineNo">244</span> final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it =<a name="line.244"></a>
+<span class="sourceLineNo">245</span> completed.entrySet().iterator();<a name="line.245"></a>
+<span class="sourceLineNo">246</span> while (it.hasNext() && store.isRunning()) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next();<a name="line.247"></a>
+<span class="sourceLineNo">248</span> final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue();<a name="line.248"></a>
+<span class="sourceLineNo">249</span> final Procedure<?> proc = retainer.getProcedure();<a name="line.249"></a>
+<span class="sourceLineNo">250</span><a name="line.250"></a>
+<span class="sourceLineNo">251</span> // TODO: Select TTL based on Procedure type<a name="line.251"></a>
+<span class="sourceLineNo">252</span> if (retainer.isExpired(now, evictTtl, evictAckTtl)) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // Failed procedures aren't persisted in WAL.<a name="line.253"></a>
+<span class="sourceLineNo">254</span> if (!(proc instanceof FailedProcedure)) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> batchIds[batchCount++] = entry.getKey();<a name="line.255"></a>
+<span class="sourceLineNo">256</span> if (batchCount == batchIds.length) {<a name="line.256"></a>
+<span class="sourceLineNo">257</span> store.delete(batchIds, 0, batchCount);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> batchCount = 0;<a name="line.258"></a>
+<span class="sourceLineNo">259</span> }<a name="line.259"></a>
+<span class="sourceLineNo">260</span> }<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final NonceKey nonceKey = proc.getNonceKey();<a name="line.261"></a>
+<span class="sourceLineNo">262</span> if (nonceKey != null) {<a name="line.262"></a>
+<span class="sourceLineNo">263</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.263"></a>
+<span class="sourceLineNo">264</span> }<a name="line.264"></a>
+<span class="sourceLineNo">265</span> it.remove();<a name="line.265"></a>
+<span class="sourceLineNo">266</span> LOG.trace("Evict completed {}", proc);<a name="line.266"></a>
+<span class="sourceLineNo">267</span> }<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span> if (batchCount > 0) {<a name="line.269"></a>
+<span class="sourceLineNo">270</span> store.delete(batchIds, 0, batchCount);<a name="line.270"></a>
+<span class="sourceLineNo">271</span> }<a name="line.271"></a>
+<span class="sourceLineNo">272</span> }<a name="line.272"></a>
+<span class="sourceLineNo">273</span> }<a name="line.273"></a>
+<span class="sourceLineNo">274</span><a name="line.274"></a>
+<span class="sourceLineNo">275</span> /**<a name="line.275"></a>
+<span class="sourceLineNo">276</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.276"></a>
+<span class="sourceLineNo">277</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.277"></a>
+<span class="sourceLineNo">278</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.278"></a>
<span class="sourceLineNo">279</span> */<a name="line.279"></a>
-<span class="sourceLineNo">280</span> private ThreadGroup threadGroup;<a name="line.280"></a>
-<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> /**<a name="line.282"></a>
-<span class="sourceLineNo">283</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.283"></a>
-<span class="sourceLineNo">284</span> * resource handling rather than observing in a #join is unexpected).<a name="line.284"></a>
-<span class="sourceLineNo">285</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.285"></a>
-<span class="sourceLineNo">286</span> * (Should be ok).<a name="line.286"></a>
+<span class="sourceLineNo">280</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.280"></a>
+<span class="sourceLineNo">281</span> new ConcurrentHashMap<>();<a name="line.281"></a>
+<span class="sourceLineNo">282</span><a name="line.282"></a>
+<span class="sourceLineNo">283</span> /**<a name="line.283"></a>
+<span class="sourceLineNo">284</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.284"></a>
+<span class="sourceLineNo">285</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.285"></a>
+<span class="sourceLineNo">286</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.286"></a>
<span class="sourceLineNo">287</span> */<a name="line.287"></a>
-<span class="sourceLineNo">288</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.288"></a>
-<span class="sourceLineNo">289</span><a name="line.289"></a>
-<span class="sourceLineNo">290</span> /**<a name="line.290"></a>
-<span class="sourceLineNo">291</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.291"></a>
-<span class="sourceLineNo">292</span> * resource handling rather than observing in a #join is unexpected).<a name="line.292"></a>
-<span class="sourceLineNo">293</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.293"></a>
-<span class="sourceLineNo">294</span> * (Should be ok).<a name="line.294"></a>
-<span class="sourceLineNo">295</span> */<a name="line.295"></a>
-<span class="sourceLineNo">296</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.296"></a>
+<span class="sourceLineNo">288</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.288"></a>
+<span class="sourceLineNo">289</span> new ConcurrentHashMap<>();<a name="line.289"></a>
+<span class="sourceLineNo">290</span><a name="line.290"></a>
+<span class="sourceLineNo">291</span> /**<a name="line.291"></a>
+<span class="sourceLineNo">292</span> * Helper map to lookup the live procedures by ID.<a name="line.292"></a>
+<span class="sourceLineNo">293</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.293"></a>
+<span class="sourceLineNo">294</span> */<a name="line.294"></a>
+<span class="sourceLineNo">295</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.295"></a>
+<span class="sourceLineNo">296</span> new ConcurrentHashMap<>();<a name="line.296"></a>
<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> private int corePoolSize;<a name="line.298"></a>
-<span class="sourceLineNo">299</span> private int maxPoolSize;<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> private volatile long keepAliveTime;<a name="line.301"></a>
-<span class="sourceLineNo">302</span><a name="line.302"></a>
-<span class="sourceLineNo">303</span> /**<a name="line.303"></a>
-<span class="sourceLineNo">304</span> * Scheduler/Queue that contains runnable procedures.<a name="line.304"></a>
-<span class="sourceLineNo">305</span> */<a name="line.305"></a>
-<span class="sourceLineNo">306</span> private final ProcedureScheduler scheduler;<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.309"></a>
-<span class="sourceLineNo">310</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.310"></a>
-<span class="sourceLineNo">311</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.311"></a>
-<span class="sourceLineNo">312</span> private final TEnvironment environment;<a name="line.312"></a>
-<span class="sourceLineNo">313</span> private final ProcedureStore store;<a name="line.313"></a>
-<span class="sourceLineNo">314</span><a name="line.314"></a>
-<span class="sourceLineNo">315</span> private final boolean checkOwnerSet;<a name="line.315"></a>
+<span class="sourceLineNo">298</span> /**<a name="line.298"></a>
+<span class="sourceLineNo">299</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.299"></a>
+<span class="sourceLineNo">300</span> * contains every root procedure.<a name="line.300"></a>
+<span class="sourceLineNo">301</span> */<a name="line.301"></a>
+<span class="sourceLineNo">302</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.302"></a>
+<span class="sourceLineNo">303</span><a name="line.303"></a>
+<span class="sourceLineNo">304</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.304"></a>
+<span class="sourceLineNo">305</span> new CopyOnWriteArrayList<>();<a name="line.305"></a>
+<span class="sourceLineNo">306</span><a name="line.306"></a>
+<span class="sourceLineNo">307</span> private Configuration conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> /**<a name="line.309"></a>
+<span class="sourceLineNo">310</span> * Created in the {@link #start(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.310"></a>
+<span class="sourceLineNo">311</span> * resource handling rather than observing in a #join is unexpected).<a name="line.311"></a>
+<span class="sourceLineNo">312</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.312"></a>
+<span class="sourceLineNo">313</span> * (Should be ok).<a name="line.313"></a>
+<span class="sourceLineNo">314</span> */<a name="line.314"></a>
+<span class="sourceLineNo">315</span> private ThreadGroup threadGroup;<a name="line.315"></a>
<span class="sourceLineNo">316</span><a name="line.316"></a>
-<span class="sourceLineNo">317</span> // To prevent concurrent execution of the same procedure.<a name="line.317"></a>
-<span class="sourceLineNo">318</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.318"></a>
-<span class="sourceLineNo">319</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.319"></a>
-<span class="sourceLineNo">320</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.320"></a>
-<span class="sourceLineNo">321</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.321"></a>
-<span class="sourceLineNo">322</span> // execution of the same procedure.<a name="line.322"></a>
-<span class="sourceLineNo">323</span> private final IdLock procExecutionLock = new IdLock();<a name="line.323"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * resource handling rather than observing in a #join is unexpected).<a name="line.319"></a>
+<span class="sourceLineNo">320</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.320"></a>
+<span class="sourceLineNo">321</span> * (Should be ok).<a name="line.321"></a>
+<span class="sourceLineNo">322</span> */<a name="line.322"></a>
+<span class="sourceLineNo">323</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.323"></a>
<span class="sourceLineNo">324</span><a name="line.324"></a>
-<span class="sourceLineNo">325</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.325"></a>
-<span class="sourceLineNo">326</span> final ProcedureStore store) {<a name="line.326"></a>
-<span class="sourceLineNo">327</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.327"></a>
-<span class="sourceLineNo">328</span> }<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.330"></a>
-<span class="sourceLineNo">331</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.331"></a>
-<span class="sourceLineNo">332</span> this.environment = environment;<a name="line.332"></a>
-<span class="sourceLineNo">333</span> this.scheduler = scheduler;<a name="line.333"></a>
-<span class="sourceLineNo">334</span> this.store = store;<a name="line.334"></a>
-<span class="sourceLineNo">335</span> this.conf = conf;<a name="line.335"></a>
-<span class="sourceLineNo">336</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> refreshConfiguration(conf);<a name="line.337"></a>
-<span class="sourceLineNo">338</span><a name="line.338"></a>
-<span class="sourceLineNo">339</span> }<a name="line.339"></a>
-<span class="sourceLineNo">340</span><a name="line.340"></a>
-<span class="sourceLineNo">341</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.341"></a>
-<span class="sourceLineNo">342</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.342"></a>
-<span class="sourceLineNo">343</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.343"></a>
-<span class="sourceLineNo">344</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.344"></a>
-<span class="sourceLineNo">345</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.345"></a>
-<span class="sourceLineNo">346</span><a name="line.346"></a>
-<span class="sourceLineNo">347</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.347"></a>
-<span class="sourceLineNo">348</span> @Override<a name="line.348"></a>
-<span class="sourceLineNo">349</span> public void setMaxProcId(long maxProcId) {<a name="line.349"></a>
-<span class="sourceLineNo">350</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.350"></a>
-<span class="sourceLineNo">351</span> lastProcId.set(maxProcId);<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span><a name="line.353"></a>
-<span class="sourceLineNo">354</span> @Override<a name="line.354"></a>
-<span class="sourceLineNo">355</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> loadProcedures(procIter, abortOnCorruption);<a name="line.356"></a>
-<span class="sourceLineNo">357</span> }<a name="line.357"></a>
-<span class="sourceLineNo">358</span><a name="line.358"></a>
-<span class="sourceLineNo">359</span> @Override<a name="line.359"></a>
-<span class="sourceLineNo">360</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.360"></a>
-<span class="sourceLineNo">361</span> int corruptedCount = 0;<a name="line.361"></a>
-<span class="sourceLineNo">362</span> while (procIter.hasNext()) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> Procedure<?> proc = procIter.next();<a name="line.363"></a>
-<span class="sourceLineNo">364</span> LOG.error("Corrupt " + proc);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> corruptedCount++;<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span> }<a name="line.370"></a>
-<span class="sourceLineNo">371</span> });<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
+<span class="sourceLineNo">325</span> /**<a name="line.325"></a>
+<span class="sourceLineNo">326</span> * Created in the {@link #start(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.326"></a>
+<span class="sourceLineNo">327</span> * resource handling rather than observing in a #join is unexpected).<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.328"></a>
+<span class="sourceLineNo">329</span> * (Should be ok).<a name="line.329"></a>
+<span class="sourceLineNo">330</span> */<a name="line.330"></a>
+<span class="sourceLineNo">331</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.331"></a>
+<span class="sourceLineNo">332</span><a name="line.332"></a>
+<span class="sourceLineNo">333</span> private int corePoolSize;<a name="line.333"></a>
+<span class="sourceLineNo">334</span> private int maxPoolSize;<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private volatile long keepAliveTime;<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> /**<a name="line.338"></a>
+<span class="sourceLineNo">339</span> * Scheduler/Queue that contains runnable procedures.<a name="line.339"></a>
+<span class="sourceLineNo">340</span> */<a name="line.340"></a>
+<span class="sourceLineNo">341</span> private final ProcedureScheduler scheduler;<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.345"></a>
+<span class="sourceLineNo">346</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.346"></a>
+<span class="sourceLineNo">347</span> private final TEnvironment environment;<a name="line.347"></a>
+<span class="sourceLineNo">348</span> private final ProcedureStore store;<a name="line.348"></a>
+<span class="sourceLineNo">349</span><a name="line.349"></a>
+<span class="sourceLineNo">350</span> private final boolean checkOwnerSet;<a name="line.350"></a>
+<span class="sourceLineNo">351</span><a name="line.351"></a>
+<span class="sourceLineNo">352</span> // To prevent concurrent execution of the same procedure.<a name="line.352"></a>
+<span class="sourceLineNo">353</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.353"></a>
+<span class="sourceLineNo">354</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.354"></a>
+<span class="sourceLineNo">355</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.355"></a>
+<span class="sourceLineNo">356</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.356"></a>
+<span class="sourceLineNo">357</span> // execution of the same procedure.<a name="line.357"></a>
+<span class="sourceLineNo">358</span> private final IdLock procExecutionLock = new IdLock();<a name="line.358"></a>
+<span class="sourceLineNo">359</span><a name="line.359"></a>
+<span class="sourceLineNo">360</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.360"></a>
+<span class="sourceLineNo">361</span> final ProcedureStore store) {<a name="line.361"></a>
+<span class="sourceLineNo">362</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.366"></a>
+<span class="sourceLineNo">367</span> this.environment = environment;<a name="line.367"></a>
+<span class="sourceLineNo">368</span> this.scheduler = scheduler;<a name="line.368"></a>
+<span class="sourceLineNo">369</span> this.store = store;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> this.conf = conf;<a name="line.370"></a>
+<span class="sourceLineNo">371</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.371"></a>
+<span class="sourceLineNo">372</span> refreshConfiguration(conf);<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
-<span class="sourceLineNo">374</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.374"></a>
-<span class="sourceLineNo">375</span> proc.restoreLock(getEnvironment());<a name="line.375"></a>
-<span class="sourceLineNo">376</span> restored.add(proc.getProcId());<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.379"></a>
-<span class="sourceLineNo">380</span> while (!stack.isEmpty()) {<a name="line.380"></a>
-<span class="sourceLineNo">381</span> restoreLock(stack.pop(), restored);<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> // Restore the locks for all the procedures.<a name="line.385"></a>
-<span class="sourceLineNo">386</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.387"></a>
-<span class="sourceLineNo">388</span> // calling the acquireLock method for the parent procedure.<a name="line.388"></a>
-<span class="sourceLineNo">389</span> // The algorithm is straight-forward:<a name="line.389"></a>
-<span class="sourceLineNo">390</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.391"></a>
-<span class="sourceLineNo">392</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // unless<a name="line.393"></a>
-<span class="sourceLineNo">394</span> // a. We have no parent, i.e, we are the root procedure<a name="line.394"></a>
-<span class="sourceLineNo">395</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.395"></a>
-<span class="sourceLineNo">396</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.396"></a>
-<span class="sourceLineNo">397</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.397"></a>
-<span class="sourceLineNo">398</span> private void restoreLocks() {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> Set<Long> restored = new HashSet<>();<a name="line.399"></a>
-<span class="sourceLineNo">400</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.400"></a>
-<span class="sourceLineNo">401</span> procedures.values().forEach(proc -> {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> for (;;) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> if (restored.contains(proc.getProcId())) {<a name="line.403"></a>
-<span class="sourceLineNo">404</span> restoreLocks(stack, restored);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> return;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> }<a name="line.406"></a>
-<span class="sourceLineNo">407</span> if (!proc.hasParent()) {<a name="line.407"></a>
-<span class="sourceLineNo">408</span> restoreLock(proc, restored);<a name="line.408"></a>
-<span class="sourceLineNo">409</span> restoreLocks(stack, restored);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> return;<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span> stack.push(proc);<a name="line.412"></a>
-<span class="sourceLineNo">413</span> proc = procedures.get(proc.getParentProcId());<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> });<a name="line.415"></a>
-<span class="sourceLineNo">416</span> }<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.418"></a>
-<span class="sourceLineNo">419</span> throws IOException {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> // 1. Build the rollback stack<a name="line.420"></a>
-<span class="sourceLineNo">421</span> int runnablesCount = 0;<a name="line.421"></a>
-<span class="sourceLineNo">422</span> int failedCount = 0;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> while (procIter.hasNext()) {<a name="line.423"></a>
-<span class="sourceLineNo">424</span> boolean finished = procIter.isNextFinished();<a name="line.424"></a>
-<span class="sourceLineNo">425</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.425"></a>
-<span class="sourceLineNo">426</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.426"></a>
-<span class="sourceLineNo">427</span> long procId = proc.getProcId();<a name="line.427"></a>
-<span class="sourceLineNo">428</span><a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (finished) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.430"></a>
-<span class="sourceLineNo">431</span> LOG.debug("Completed {}", proc);<a name="line.431"></a>
-<span class="sourceLineNo">432</span> } else {<a name="line.432"></a>
-<span class="sourceLineNo">433</span> if (!proc.hasParent()) {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.434"></a>
-<span class="sourceLineNo">435</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.435"></a>
-<span class="sourceLineNo">436</span> }<a name="line.436"></a>
-<span class="sourceLineNo">437</span><a name="line.437"></a>
-<span class="sourceLineNo">438</span> // add the procedure to the map<a name="line.438"></a>
-<span class="sourceLineNo">439</span> proc.beforeReplay(getEnvironment());<a name="line.439"></a>
-<span class="sourceLineNo">440</span> procedures.put(proc.getProcId(), proc);<a name="line.440"></a>
-<span class="sourceLineNo">441</span><a name="line.441"></a>
-<span class="sourceLineNo">442</span> if (proc.getState() == ProcedureState.RUNNABLE) {<a name="line.442"></a>
-<span class="sourceLineNo">443</span> runnablesCount++;<a name="line.443"></a>
-<span class="sourceLineNo">444</span> } else if (proc.getState() == ProcedureState.FAILED) {<a name="line.444"></a>
-<span class="sourceLineNo">445</span> failedCount++;<a name="line.445"></a>
+<span class="sourceLineNo">374</span> }<a name="line.374"></a>
+<span class="sourceLineNo">375</span><a name="line.375"></a>
+<span class="sourceLineNo">376</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.376"></a>
+<span class="sourceLineNo">377</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.377"></a>
+<span class="sourceLineNo">378</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.378"></a>
+<span class="sourceLineNo">379</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.379"></a>
+<span class="sourceLineNo">380</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.380"></a>
+<span class="sourceLineNo">381</span><a name="line.381"></a>
+<span class="sourceLineNo">382</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> @Override<a name="line.383"></a>
+<span class="sourceLineNo">384</span> public void setMaxProcId(long maxProcId) {<a name="line.384"></a>
+<span class="sourceLineNo">385</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.385"></a>
+<span class="sourceLineNo">386</span> lastProcId.set(maxProcId);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
+<span class="sourceLineNo">388</span><a name="line.388"></a>
+<span class="sourceLineNo">389</span> @Override<a name="line.389"></a>
+<span class="sourceLineNo">390</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.390"></a>
+<span class="sourceLineNo">391</span> loadProcedures(procIter, abortOnCorruption);<a name="line.391"></a>
+<span class="sourceLineNo">392</span> }<a name="line.392"></a>
+<span class="sourceLineNo">393</span><a name="line.393"></a>
+<span class="sourceLineNo">394</span> @Override<a name="line.394"></a>
+<span class="sourceLineNo">395</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.395"></a>
+<span class="sourceLineNo">396</span> int corruptedCount = 0;<a name="line.396"></a>
+<span class="sourceLineNo">397</span> while (procIter.hasNext()) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> Procedure<?> proc = procIter.next();<a name="line.398"></a>
+<span class="sourceLineNo">399</span> LOG.error("Corrupt " + proc);<a name="line.399"></a>
+<span class="sourceLineNo">400</span> corruptedCount++;<a name="line.400"></a>
+<span class="sourceLineNo">401</span> }<a name="line.401"></a>
+<span class="sourceLineNo">402</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.403"></a>
+<span class="sourceLineNo">404</span> }<a name="line.404"></a>
+<span class="sourceLineNo">405</span> }<a name="line.405"></a>
+<span class="sourceLineNo">406</span> });<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> proc.restoreLock(getEnvironment());<a name="line.410"></a>
+<span class="sourceLineNo">411</span> restored.add(proc.getProcId());<a name="line.411"></a>
+<span class="sourceLineNo">412</span> }<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> while (!stack.isEmpty()) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> restoreLock(stack.pop(), restored);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> // Restore the locks for all the procedures.<a name="line.420"></a>
+<span class="sourceLineNo">421</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.421"></a>
+<span class="sourceLineNo">422</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.422"></a>
+<span class="sourceLineNo">423</span> // calling the acquireLock method for the parent procedure.<a name="line.423"></a>
+<span class="sourceLineNo">424</span> // The algorithm is straight-forward:<a name="line.424"></a>
+<span class="sourceLineNo">425</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.425"></a>
+<span class="sourceLineNo">426</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.426"></a>
+<span class="sourceLineNo">427</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.427"></a>
+<span class="sourceLineNo">428</span> // unless<a name="line.428"></a>
+<span class="sourceLineNo">429</span> // a. We have no parent, i.e, we are the root procedure<a name="line.429"></a>
+<span class="sourceLineNo">430</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.430"></a>
+<span class="sourceLineNo">431</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.431"></a>
+<span class="sourceLineNo">432</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.432"></a>
+<span class="sourceLineNo">433</span> private void restoreLocks() {<a name="line.433"></a>
+<span class="sourceLineNo">434</span> Set<Long> restored = new HashSet<>();<a name="line.434"></a>
+<span class="sourceLineNo">435</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.435"></a>
+<span class="sourceLineNo">436</span> procedures.values().forEach(proc -> {<a name="line.436"></a>
+<span class="sourceLineNo">437</span> for (;;) {<a name="line.437"></a>
+<span class="sourceLineNo">438</span> if (restored.contains(proc.getProcId())) {<a name="line.438"></a>
+<span class="sourceLineNo">439</span> restoreLocks(stack, restored);<a name="line.439"></a>
+<span class="sourceLineNo">440</span> return;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> }<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (!proc.hasParent()) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> restoreLock(proc, restored);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> restoreLocks(stack, restored);<a name="line.444"></a>
+<span class="sourceLineNo">445</span> return;<a name="line.445"></a>
<span class="sourceLineNo">446</span> }<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span><a name="line.448"></a>
-<span class="sourceLineNo">449</span> // add the nonce to the map<a name="line.449"></a>
-<span class="sourceLineNo">450</span> if (nonceKey != null) {<a name="line.450"></a>
-<span class="sourceLineNo">451</span> nonceKeysToProcIdsMap.put(nonceKey, procId);<a name="line.451"></a>
-<span class="sourceLineNo">452</span> }<a name="line.452"></a>
-<span class="sourceLineNo">453</span> }<a name="line.453"></a>
-<span class="sourceLineNo">454</span><a name="line.454"></a>
-<span class="sourceLineNo">455</span> // 2. Initialize the stacks<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // In the old implementation, for procedures in FAILED state, we will push it into the<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // ProcedureScheduler directly to execute the rollback. But this does not work after we<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // introduce the restore lock stage.<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // For now, when we acquire a xlock, we will remove the queue from runQueue in scheduler, and<a name="line.459"></a>
-<span class="sourceLineNo">460</span> // then when a procedure which has lock access, for example, a sub procedure of the procedure<a name="line.460"></a>
-<span class="sourceLineNo">461</span> // which has the xlock, is pushed into the scheduler, we will add the queue back to let the<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // workers poll from it. The assumption here is that, the procedure which has the xlock should<a name="line.462"></a>
-<span class="sourceLineNo">463</span> // have been polled out already, so when loading we can not add the procedure to scheduler first<a name="line.463"></a>
-<span class="sourceLineNo">464</span> // and then call acquireLock, since the procedure is still in the queue, and since we will<a name="line.464"></a>
-<span class="sourceLineNo">465</span> // remove the queue from runQueue, then no one can poll it out, then there is a dead lock<a name="line.465"></a>
-<span class="sourceLineNo">466</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnablesCount);<a name="line.466"></a>
-<span class="sourceLineNo">467</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> Set<Procedure<TEnvironment>> waitingSet = null;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> procIter.reset();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> while (procIter.hasNext()) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> if (procIter.isNextFinished()) {<a name="line.471"></a>
-<span class="sourceLineNo">472</span> procIter.skipNext();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> continue;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> }<a name="line.474"></a>
-<span class="sourceLineNo">475</span><a name="line.475"></a>
-<span class="sourceLineNo">476</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> LOG.debug("Loading {}", proc);<a name="line.479"></a>
-<span class="sourceLineNo">480</span><a name="line.480"></a>
-<span class="sourceLineNo">481</span> Long rootProcId = getRootProcedureId(proc);<a name="line.481"></a>
-<span class="sourceLineNo">482</span> if (rootProcId == null) {<a name="line.482"></a>
-<span class="sourceLineNo">483</span> // The 'proc' was ready to run but the root procedure was rolledback?<a name="line.483"></a>
-<span class="sourceLineNo">484</span> scheduler.addBack(proc);<a name="line.484"></a>
-<span class="sourceLineNo">485</span> continue;<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span><a name="line.487"></a>
-<span class="sourceLineNo">488</span> if (proc.hasParent()) {<a name="line.488"></a>
-<span class="sourceLineNo">489</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // corrupted procedures are handled later at step 3<a name="line.490"></a>
-<span class="sourceLineNo">491</span> if (parent != null && !proc.isFinished()) {<a name="line.491"></a>
-<span class="sourceLineNo">492</span> parent.incChildrenLatch();<a name="line.492"></a>
-<span class="sourceLineNo">493</span> }<a name="line.493"></a>
-<span class="sourceLineNo">494</span> }<a name="line.494"></a>
-<span class="sourceLineNo">495</span><a name="line.495"></a>
-<span class="sourceLineNo">496</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.496"></a>
-<span class="sourceLineNo">497</span> procStack.loadStack(proc);<a name="line.497"></a>
-<span class="sourceLineNo">498</span><a name="line.498"></a>
-<span class="sourceLineNo">499</span> proc.setRootProcId(rootProcId);<a name="line.499"></a>
-<span class="sourceLineNo">500</span> switch (proc.getState()) {<a name="line.500"></a>
-<span class="sourceLineNo">501</span> case RUNNABLE:<a name="line.501"></a>
-<span class="sourceLineNo">502</span> runnableList.add(proc);<a name="line.502"></a>
-<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case WAITING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> if (!proc.hasChildren()) {<a name="line.505"></a>
-<span class="sourceLineNo">506</span> runnableList.add(proc);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> }<a name="line.507"></a>
-<span class="sourceLineNo">508</span> break;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> case WAITING_TIMEOUT:<a name="line.509"></a>
-<span class="sourceLineNo">510</span> if (waitingSet == null) {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> waitingSet = new HashSet<>();<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span> waitingSet.add(proc);<a name="line.513"></a>
-<span class="sourceLineNo">514</span> break;<a name="line.514"></a>
-<span class="sourceLineNo">515</span> case FAILED:<a name="line.515"></a>
-<span class="sourceLineNo">516</span> failedList.add(proc);<a name="line.516"></a>
-<span class="sourceLineNo">517</span> break;<a name="line.517"></a>
-<span class="sourceLineNo">518</span> case ROLLEDBACK:<a name="line.518"></a>
-<span class="sourceLineNo">519</span> case INITIALIZING:<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.520"></a>
-<span class="sourceLineNo">521</span> LOG.error(msg);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> throw new UnsupportedOperationException(msg);<a name="line.522"></a>
-<span class="sourceLineNo">523</span> default:<a name="line.523"></a>
-<span class="sourceLineNo">524</span> break;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span><a name="line.527"></a>
-<span class="sourceLineNo">528</span> // 3. Validate the stacks<a name="line.528"></a>
-<span class="sourceLineNo">529</span> int corruptedCount = 0;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> Iterator<Map.Entry<Long, RootProcedureState<TEnvironment>>> itStack =<a name="line.530"></a>
-<span class="sourceLineNo">531</span> rollbackStack.entrySet().iterator();<a name="line.531"></a>
-<span class="sourceLineNo">532</span> while (itStack.hasNext()) {<a name="line.532"></a>
-<span class="sourceLineNo">533</span> Map.Entry<Long, RootProcedureState<TEnvironment>> entry = itStack.next();<a name="line.533"></a>
-<span class="sourceLineNo">534</span> RootProcedureState<TEnvironment> procStack = entry.getValue();<a name="line.534"></a>
-<span class="sourceLineNo">535</span> if (procStack.isValid()) continue;<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> for (Procedure<TEnvironment> proc : procStack.getSubproceduresStack()) {<a name="line.537"></a>
-<span class="sourceLineNo">538</span> LOG.error("Corrupted " + proc);<a name="line.538"></a>
-<span class="sourceLineNo">539</span> procedures.remove(proc.getProcId());<a name="line.539"></a>
-<span class="sourceLineNo">540</span> runnableList.remove(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> if (waitingSet != null) waitingSet.remove(proc);<a name="line.541"></a>
-<span class="sourceLineNo">542</span> corruptedCount++;<a name="line.542"></a>
-<span class="sourceLineNo">543</span> }<a name="line.543"></a>
-<span class="sourceLineNo">544</span> itStack.remove();<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span><a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> throw new IOException("found " + corruptedCount + " procedures on replay");<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span><a name="line.550"></a>
-<span class="sourceLineNo">551</span> // 4. Push the procedures to the timeout executor<a name="line.551"></a>
-<span class="sourceLineNo">552</span> if (waitingSet != null && !waitingSet.isEmpty()) {<a name="line.552"></a>
-<span class="sourceLineNo">553</span> for (Procedure<TEnvironment> proc: waitingSet) {<a name="line.553"></a>
-<span class="sourceLineNo">554</span> proc.afterReplay(getEnvironment());<a name="line.554"></a>
-<span class="sourceLineNo">555</span> timeoutExecutor.add(proc);<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span> }<a name="line.557"></a>
-<span class="sourceLineNo">558</span> // 5. restore locks<a name="line.558"></a>
-<span class="sourceLineNo">559</span> restoreLocks();<a name="line.559"></a>
-<span class="sourceLineNo">560</span> // 6. Push the procedure to the scheduler<a name="line.560"></a>
-<span class="sourceLineNo">561</span> failedList.forEach(scheduler::addBack);<a name="line.561"></a>
-<span class="sourceLineNo">562</span> runnableList.forEach(p -> {<a name="line.562"></a>
-<span class="sourceLineNo">563</span> p.afterReplay(getEnvironment());<a name="line.563"></a>
-<span class="sourceLineNo">564</span> if (!p.hasParent()) {<a name="line.564"></a>
-<span class="sourceLineNo">565</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.565"></a>
-<span class="sourceLineNo">566</span> }<a name="line.566"></a>
-<span class="sourceLineNo">567</span> scheduler.addBack(p);<a name="line.567"></a>
-<span class="sourceLineNo">568</span> });<a name="line.568"></a>
-<span class="sourceLineNo">569</span> }<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> /**<a name="line.571"></a>
-<span class="sourceLineNo">572</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.572"></a>
-<span class="sourceLineNo">573</span> * <p/><a name="line.573"></a>
-<span class="sourceLineNo">574</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.574"></a>
-<span class="sourceLineNo">575</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.575"></a>
-<span class="sourceLineNo">576</span> * pending and in-progress procedures.<a name="line.576"></a>
-<span class="sourceLineNo">577</span> * @param numThreads number of threads available for procedure execution.<a name="line.577"></a>
-<span class="sourceLineNo">578</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.578"></a>
-<span class="sourceLineNo">579</span> * is found on replay. otherwise false.<a name="line.579"></a>
-<span class="sourceLineNo">580</span> */<a name="line.580"></a>
-<span class="sourceLineNo">581</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.581"></a>
-<span class="sourceLineNo">582</span> // We have numThreads executor + one timer thread used for timing out<a name="line.582"></a>
-<span class="sourceLineNo">583</span> // procedures and triggering periodic procedures.<a name="line.583"></a>
-<span class="sourceLineNo">584</span> this.corePoolSize = numThreads;<a name="line.584"></a>
-<span class="sourceLineNo">585</span> this.maxPoolSize = 10 * numThreads;<a name="line.585"></a>
-<span class="sourceLineNo">586</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.586"></a>
-<span class="sourceLineNo">587</span> corePoolSize, maxPoolSize);<a name="line.587"></a>
-<span class="sourceLineNo">588</span><a name="line.588"></a>
-<span class="sourceLineNo">589</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.589"></a>
-<span class="sourceLineNo">590</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup);<a name="line.590"></a>
-<span class="sourceLineNo">591</span><a name="line.591"></a>
-<span class="sourceLineNo">592</span> // Create the workers<a name="line.592"></a>
-<span class="sourceLineNo">593</span> workerId.set(0);<a name="line.593"></a>
-<span class="sourceLineNo">594</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.594"></a>
-<span class="sourceLineNo">595</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.595"></a>
-<span class="sourceLineNo">596</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.596"></a>
-<span class="sourceLineNo">597</span> }<a name="line.597"></a>
-<span class="sourceLineNo">598</span><a name="line.598"></a>
-<span class="sourceLineNo">599</span> long st, et;<a name="line.599"></a>
-<span class="sourceLineNo">600</span><a name="line.600"></a>
-<span class="sourceLineNo">601</span> // Acquire the store lease.<a name
<TRUNCATED>
[02/20] hbase-site git commit: Published site at
1114a1a65e78a00f643ef15b9a13bc710cb80845.
Posted by gi...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/3b0a06ec/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
index 45dd2e4..4310a16 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.html
@@ -175,399 +175,406 @@
<span class="sourceLineNo">167</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.167"></a>
<span class="sourceLineNo">168</span> }<a name="line.168"></a>
<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.170"></a>
-<span class="sourceLineNo">171</span> boolean value) {<a name="line.171"></a>
-<span class="sourceLineNo">172</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.172"></a>
-<span class="sourceLineNo">173</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.173"></a>
+<span class="sourceLineNo">170</span> public static <TEnv> void toggleKillAfterStoreUpdate(ProcedureExecutor<TEnv> procExecutor) {<a name="line.170"></a>
+<span class="sourceLineNo">171</span> createExecutorTesting(procExecutor);<a name="line.171"></a>
+<span class="sourceLineNo">172</span> procExecutor.testing.killAfterStoreUpdate = !procExecutor.testing.killAfterStoreUpdate;<a name="line.172"></a>
+<span class="sourceLineNo">173</span> LOG.warn("Set Kill after store update to: " + procExecutor.testing.killAfterStoreUpdate);<a name="line.173"></a>
<span class="sourceLineNo">174</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.174"></a>
<span class="sourceLineNo">175</span> }<a name="line.175"></a>
<span class="sourceLineNo">176</span><a name="line.176"></a>
-<span class="sourceLineNo">177</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.177"></a>
-<span class="sourceLineNo">178</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.178"></a>
-<span class="sourceLineNo">179</span> if (procExecutor.testing == null) return;<a name="line.179"></a>
-<span class="sourceLineNo">180</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.180"></a>
-<span class="sourceLineNo">181</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.181"></a>
-<span class="sourceLineNo">182</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.182"></a>
-<span class="sourceLineNo">183</span> 1, procExecutor.getCorePoolSize());<a name="line.183"></a>
-<span class="sourceLineNo">184</span> }<a name="line.184"></a>
-<span class="sourceLineNo">185</span> }<a name="line.185"></a>
-<span class="sourceLineNo">186</span><a name="line.186"></a>
-<span class="sourceLineNo">187</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.187"></a>
-<span class="sourceLineNo">188</span> throws IOException {<a name="line.188"></a>
-<span class="sourceLineNo">189</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.189"></a>
-<span class="sourceLineNo">190</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.190"></a>
-<span class="sourceLineNo">191</span> procStore.start(1);<a name="line.191"></a>
-<span class="sourceLineNo">192</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.192"></a>
-<span class="sourceLineNo">193</span> try {<a name="line.193"></a>
-<span class="sourceLineNo">194</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.194"></a>
-<span class="sourceLineNo">195</span> } finally {<a name="line.195"></a>
-<span class="sourceLineNo">196</span> procStore.stop(false);<a name="line.196"></a>
-<span class="sourceLineNo">197</span> procExecutor.stop();<a name="line.197"></a>
-<span class="sourceLineNo">198</span> }<a name="line.198"></a>
-<span class="sourceLineNo">199</span> }<a name="line.199"></a>
-<span class="sourceLineNo">200</span><a name="line.200"></a>
-<span class="sourceLineNo">201</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.201"></a>
-<span class="sourceLineNo">202</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.202"></a>
-<span class="sourceLineNo">203</span> }<a name="line.203"></a>
-<span class="sourceLineNo">204</span><a name="line.204"></a>
-<span class="sourceLineNo">205</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.205"></a>
-<span class="sourceLineNo">206</span> final long nonceGroup, final long nonce) {<a name="line.206"></a>
-<span class="sourceLineNo">207</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.207"></a>
-<span class="sourceLineNo">208</span> waitProcedure(procExecutor, procId);<a name="line.208"></a>
-<span class="sourceLineNo">209</span> return procId;<a name="line.209"></a>
+<span class="sourceLineNo">177</span> public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor,<a name="line.177"></a>
+<span class="sourceLineNo">178</span> boolean value) {<a name="line.178"></a>
+<span class="sourceLineNo">179</span> ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value);<a name="line.179"></a>
+<span class="sourceLineNo">180</span> ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value);<a name="line.180"></a>
+<span class="sourceLineNo">181</span> assertSingleExecutorForKillTests(procExecutor);<a name="line.181"></a>
+<span class="sourceLineNo">182</span> }<a name="line.182"></a>
+<span class="sourceLineNo">183</span><a name="line.183"></a>
+<span class="sourceLineNo">184</span> private static <TEnv> void assertSingleExecutorForKillTests(<a name="line.184"></a>
+<span class="sourceLineNo">185</span> final ProcedureExecutor<TEnv> procExecutor) {<a name="line.185"></a>
+<span class="sourceLineNo">186</span> if (procExecutor.testing == null) return;<a name="line.186"></a>
+<span class="sourceLineNo">187</span> if (procExecutor.testing.killBeforeStoreUpdate ||<a name="line.187"></a>
+<span class="sourceLineNo">188</span> procExecutor.testing.toggleKillBeforeStoreUpdate) {<a name="line.188"></a>
+<span class="sourceLineNo">189</span> assertEquals("expected only one executor running during test with kill/restart",<a name="line.189"></a>
+<span class="sourceLineNo">190</span> 1, procExecutor.getCorePoolSize());<a name="line.190"></a>
+<span class="sourceLineNo">191</span> }<a name="line.191"></a>
+<span class="sourceLineNo">192</span> }<a name="line.192"></a>
+<span class="sourceLineNo">193</span><a name="line.193"></a>
+<span class="sourceLineNo">194</span> public static <TEnv> long submitAndWait(Configuration conf, TEnv env, Procedure<TEnv> proc)<a name="line.194"></a>
+<span class="sourceLineNo">195</span> throws IOException {<a name="line.195"></a>
+<span class="sourceLineNo">196</span> NoopProcedureStore procStore = new NoopProcedureStore();<a name="line.196"></a>
+<span class="sourceLineNo">197</span> ProcedureExecutor<TEnv> procExecutor = new ProcedureExecutor<>(conf, env, procStore);<a name="line.197"></a>
+<span class="sourceLineNo">198</span> procStore.start(1);<a name="line.198"></a>
+<span class="sourceLineNo">199</span> initAndStartWorkers(procExecutor, 1, false);<a name="line.199"></a>
+<span class="sourceLineNo">200</span> try {<a name="line.200"></a>
+<span class="sourceLineNo">201</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.201"></a>
+<span class="sourceLineNo">202</span> } finally {<a name="line.202"></a>
+<span class="sourceLineNo">203</span> procStore.stop(false);<a name="line.203"></a>
+<span class="sourceLineNo">204</span> procExecutor.stop();<a name="line.204"></a>
+<span class="sourceLineNo">205</span> }<a name="line.205"></a>
+<span class="sourceLineNo">206</span> }<a name="line.206"></a>
+<span class="sourceLineNo">207</span><a name="line.207"></a>
+<span class="sourceLineNo">208</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.208"></a>
+<span class="sourceLineNo">209</span> return submitAndWait(procExecutor, proc, HConstants.NO_NONCE, HConstants.NO_NONCE);<a name="line.209"></a>
<span class="sourceLineNo">210</span> }<a name="line.210"></a>
<span class="sourceLineNo">211</span><a name="line.211"></a>
-<span class="sourceLineNo">212</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
+<span class="sourceLineNo">212</span> public static <TEnv> long submitAndWait(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.212"></a>
<span class="sourceLineNo">213</span> final long nonceGroup, final long nonce) {<a name="line.213"></a>
-<span class="sourceLineNo">214</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.214"></a>
-<span class="sourceLineNo">215</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.215"></a>
-<span class="sourceLineNo">216</span> assertFalse(procId >= 0);<a name="line.216"></a>
-<span class="sourceLineNo">217</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.217"></a>
-<span class="sourceLineNo">218</span> }<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.220"></a>
-<span class="sourceLineNo">221</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.221"></a>
-<span class="sourceLineNo">222</span> Threads.sleepWithoutInterrupt(250);<a name="line.222"></a>
-<span class="sourceLineNo">223</span> }<a name="line.223"></a>
-<span class="sourceLineNo">224</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.224"></a>
+<span class="sourceLineNo">214</span> long procId = submitProcedure(procExecutor, proc, nonceGroup, nonce);<a name="line.214"></a>
+<span class="sourceLineNo">215</span> waitProcedure(procExecutor, procId);<a name="line.215"></a>
+<span class="sourceLineNo">216</span> return procId;<a name="line.216"></a>
+<span class="sourceLineNo">217</span> }<a name="line.217"></a>
+<span class="sourceLineNo">218</span><a name="line.218"></a>
+<span class="sourceLineNo">219</span> public static <TEnv> long submitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc,<a name="line.219"></a>
+<span class="sourceLineNo">220</span> final long nonceGroup, final long nonce) {<a name="line.220"></a>
+<span class="sourceLineNo">221</span> final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce);<a name="line.221"></a>
+<span class="sourceLineNo">222</span> long procId = procExecutor.registerNonce(nonceKey);<a name="line.222"></a>
+<span class="sourceLineNo">223</span> assertFalse(procId >= 0);<a name="line.223"></a>
+<span class="sourceLineNo">224</span> return procExecutor.submitProcedure(proc, nonceKey);<a name="line.224"></a>
<span class="sourceLineNo">225</span> }<a name="line.225"></a>
<span class="sourceLineNo">226</span><a name="line.226"></a>
-<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.227"></a>
-<span class="sourceLineNo">228</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.228"></a>
+<span class="sourceLineNo">227</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, Procedure proc) {<a name="line.227"></a>
+<span class="sourceLineNo">228</span> while (proc.getState() == ProcedureState.INITIALIZING) {<a name="line.228"></a>
<span class="sourceLineNo">229</span> Threads.sleepWithoutInterrupt(250);<a name="line.229"></a>
<span class="sourceLineNo">230</span> }<a name="line.230"></a>
-<span class="sourceLineNo">231</span> }<a name="line.231"></a>
-<span class="sourceLineNo">232</span><a name="line.232"></a>
-<span class="sourceLineNo">233</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.233"></a>
-<span class="sourceLineNo">234</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.234"></a>
-<span class="sourceLineNo">235</span> waitProcedure(procExecutor, procIds[i]);<a name="line.235"></a>
-<span class="sourceLineNo">236</span> }<a name="line.236"></a>
-<span class="sourceLineNo">237</span> }<a name="line.237"></a>
-<span class="sourceLineNo">238</span><a name="line.238"></a>
-<span class="sourceLineNo">239</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.239"></a>
-<span class="sourceLineNo">240</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.240"></a>
-<span class="sourceLineNo">241</span> waitProcedure(procExecutor, procId);<a name="line.241"></a>
-<span class="sourceLineNo">242</span> }<a name="line.242"></a>
-<span class="sourceLineNo">243</span> }<a name="line.243"></a>
-<span class="sourceLineNo">244</span><a name="line.244"></a>
-<span class="sourceLineNo">245</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.245"></a>
-<span class="sourceLineNo">246</span> int stableRuns = 0;<a name="line.246"></a>
-<span class="sourceLineNo">247</span> while (stableRuns < 10) {<a name="line.247"></a>
-<span class="sourceLineNo">248</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.248"></a>
-<span class="sourceLineNo">249</span> stableRuns = 0;<a name="line.249"></a>
-<span class="sourceLineNo">250</span> Threads.sleepWithoutInterrupt(100);<a name="line.250"></a>
-<span class="sourceLineNo">251</span> } else {<a name="line.251"></a>
-<span class="sourceLineNo">252</span> stableRuns++;<a name="line.252"></a>
-<span class="sourceLineNo">253</span> Threads.sleepWithoutInterrupt(25);<a name="line.253"></a>
-<span class="sourceLineNo">254</span> }<a name="line.254"></a>
-<span class="sourceLineNo">255</span> }<a name="line.255"></a>
-<span class="sourceLineNo">256</span> }<a name="line.256"></a>
-<span class="sourceLineNo">257</span><a name="line.257"></a>
-<span class="sourceLineNo">258</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.258"></a>
-<span class="sourceLineNo">259</span> long procId) {<a name="line.259"></a>
-<span class="sourceLineNo">260</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.260"></a>
-<span class="sourceLineNo">261</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span><a name="line.263"></a>
-<span class="sourceLineNo">264</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.264"></a>
-<span class="sourceLineNo">265</span> long procId) {<a name="line.265"></a>
-<span class="sourceLineNo">266</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.266"></a>
-<span class="sourceLineNo">267</span> assertTrue("expected procedure result", result != null);<a name="line.267"></a>
-<span class="sourceLineNo">268</span> assertProcNotFailed(result);<a name="line.268"></a>
+<span class="sourceLineNo">231</span> waitProcedure(procExecutor, proc.getProcId());<a name="line.231"></a>
+<span class="sourceLineNo">232</span> }<a name="line.232"></a>
+<span class="sourceLineNo">233</span><a name="line.233"></a>
+<span class="sourceLineNo">234</span> public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) {<a name="line.234"></a>
+<span class="sourceLineNo">235</span> while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) {<a name="line.235"></a>
+<span class="sourceLineNo">236</span> Threads.sleepWithoutInterrupt(250);<a name="line.236"></a>
+<span class="sourceLineNo">237</span> }<a name="line.237"></a>
+<span class="sourceLineNo">238</span> }<a name="line.238"></a>
+<span class="sourceLineNo">239</span><a name="line.239"></a>
+<span class="sourceLineNo">240</span> public static <TEnv> void waitProcedures(ProcedureExecutor<TEnv> procExecutor, long... procIds) {<a name="line.240"></a>
+<span class="sourceLineNo">241</span> for (int i = 0; i < procIds.length; ++i) {<a name="line.241"></a>
+<span class="sourceLineNo">242</span> waitProcedure(procExecutor, procIds[i]);<a name="line.242"></a>
+<span class="sourceLineNo">243</span> }<a name="line.243"></a>
+<span class="sourceLineNo">244</span> }<a name="line.244"></a>
+<span class="sourceLineNo">245</span><a name="line.245"></a>
+<span class="sourceLineNo">246</span> public static <TEnv> void waitAllProcedures(ProcedureExecutor<TEnv> procExecutor) {<a name="line.246"></a>
+<span class="sourceLineNo">247</span> for (long procId : procExecutor.getActiveProcIds()) {<a name="line.247"></a>
+<span class="sourceLineNo">248</span> waitProcedure(procExecutor, procId);<a name="line.248"></a>
+<span class="sourceLineNo">249</span> }<a name="line.249"></a>
+<span class="sourceLineNo">250</span> }<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span> public static <TEnv> void waitNoProcedureRunning(ProcedureExecutor<TEnv> procExecutor) {<a name="line.252"></a>
+<span class="sourceLineNo">253</span> int stableRuns = 0;<a name="line.253"></a>
+<span class="sourceLineNo">254</span> while (stableRuns < 10) {<a name="line.254"></a>
+<span class="sourceLineNo">255</span> if (procExecutor.getActiveExecutorCount() > 0 || procExecutor.getScheduler().size() > 0) {<a name="line.255"></a>
+<span class="sourceLineNo">256</span> stableRuns = 0;<a name="line.256"></a>
+<span class="sourceLineNo">257</span> Threads.sleepWithoutInterrupt(100);<a name="line.257"></a>
+<span class="sourceLineNo">258</span> } else {<a name="line.258"></a>
+<span class="sourceLineNo">259</span> stableRuns++;<a name="line.259"></a>
+<span class="sourceLineNo">260</span> Threads.sleepWithoutInterrupt(25);<a name="line.260"></a>
+<span class="sourceLineNo">261</span> }<a name="line.261"></a>
+<span class="sourceLineNo">262</span> }<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span> public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor,<a name="line.265"></a>
+<span class="sourceLineNo">266</span> long procId) {<a name="line.266"></a>
+<span class="sourceLineNo">267</span> assertFalse("expected a running proc", procExecutor.isFinished(procId));<a name="line.267"></a>
+<span class="sourceLineNo">268</span> assertEquals(null, procExecutor.getResult(procId));<a name="line.268"></a>
<span class="sourceLineNo">269</span> }<a name="line.269"></a>
<span class="sourceLineNo">270</span><a name="line.270"></a>
-<span class="sourceLineNo">271</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.271"></a>
-<span class="sourceLineNo">272</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.272"></a>
-<span class="sourceLineNo">273</span> }<a name="line.273"></a>
-<span class="sourceLineNo">274</span><a name="line.274"></a>
-<span class="sourceLineNo">275</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.275"></a>
-<span class="sourceLineNo">276</span> final long procId) {<a name="line.276"></a>
-<span class="sourceLineNo">277</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.277"></a>
-<span class="sourceLineNo">278</span> assertTrue("expected procedure result", result != null);<a name="line.278"></a>
-<span class="sourceLineNo">279</span> return assertProcFailed(result);<a name="line.279"></a>
+<span class="sourceLineNo">271</span> public static <TEnv> void assertProcNotFailed(ProcedureExecutor<TEnv> procExecutor,<a name="line.271"></a>
+<span class="sourceLineNo">272</span> long procId) {<a name="line.272"></a>
+<span class="sourceLineNo">273</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.273"></a>
+<span class="sourceLineNo">274</span> assertTrue("expected procedure result", result != null);<a name="line.274"></a>
+<span class="sourceLineNo">275</span> assertProcNotFailed(result);<a name="line.275"></a>
+<span class="sourceLineNo">276</span> }<a name="line.276"></a>
+<span class="sourceLineNo">277</span><a name="line.277"></a>
+<span class="sourceLineNo">278</span> public static void assertProcNotFailed(final Procedure<?> result) {<a name="line.278"></a>
+<span class="sourceLineNo">279</span> assertFalse("found exception: " + result.getException(), result.isFailed());<a name="line.279"></a>
<span class="sourceLineNo">280</span> }<a name="line.280"></a>
<span class="sourceLineNo">281</span><a name="line.281"></a>
-<span class="sourceLineNo">282</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.282"></a>
-<span class="sourceLineNo">283</span> assertEquals(true, result.isFailed());<a name="line.283"></a>
-<span class="sourceLineNo">284</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.284"></a>
-<span class="sourceLineNo">285</span> return getExceptionCause(result);<a name="line.285"></a>
-<span class="sourceLineNo">286</span> }<a name="line.286"></a>
-<span class="sourceLineNo">287</span><a name="line.287"></a>
-<span class="sourceLineNo">288</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.288"></a>
-<span class="sourceLineNo">289</span> Throwable cause = assertProcFailed(result);<a name="line.289"></a>
-<span class="sourceLineNo">290</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.290"></a>
-<span class="sourceLineNo">291</span> }<a name="line.291"></a>
-<span class="sourceLineNo">292</span><a name="line.292"></a>
-<span class="sourceLineNo">293</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.293"></a>
-<span class="sourceLineNo">294</span> Throwable cause = assertProcFailed(result);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> }<a name="line.296"></a>
-<span class="sourceLineNo">297</span><a name="line.297"></a>
-<span class="sourceLineNo">298</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.298"></a>
-<span class="sourceLineNo">299</span> Throwable cause = assertProcFailed(result);<a name="line.299"></a>
-<span class="sourceLineNo">300</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.300"></a>
-<span class="sourceLineNo">301</span> cause instanceof IllegalArgumentIOException);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> }<a name="line.302"></a>
-<span class="sourceLineNo">303</span><a name="line.303"></a>
-<span class="sourceLineNo">304</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.304"></a>
-<span class="sourceLineNo">305</span> assert procInfo.isFailed();<a name="line.305"></a>
-<span class="sourceLineNo">306</span> Throwable cause = procInfo.getException().getCause();<a name="line.306"></a>
-<span class="sourceLineNo">307</span> return cause == null ? procInfo.getException() : cause;<a name="line.307"></a>
-<span class="sourceLineNo">308</span> }<a name="line.308"></a>
-<span class="sourceLineNo">309</span><a name="line.309"></a>
-<span class="sourceLineNo">310</span> /**<a name="line.310"></a>
-<span class="sourceLineNo">311</span> * Run through all procedure flow states TWICE while also restarting<a name="line.311"></a>
-<span class="sourceLineNo">312</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.312"></a>
-<span class="sourceLineNo">313</span> *<a name="line.313"></a>
-<span class="sourceLineNo">314</span> *<p>It does<a name="line.314"></a>
-<span class="sourceLineNo">315</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.315"></a>
-<span class="sourceLineNo">316</span> * <li>Restart executor/store<a name="line.316"></a>
-<span class="sourceLineNo">317</span> * <li>Execute step N - and then save to store<a name="line.317"></a>
-<span class="sourceLineNo">318</span> * </ol><a name="line.318"></a>
-<span class="sourceLineNo">319</span> *<a name="line.319"></a>
-<span class="sourceLineNo">320</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.320"></a>
-<span class="sourceLineNo">321</span> * idempotent.<a name="line.321"></a>
-<span class="sourceLineNo">322</span> */<a name="line.322"></a>
-<span class="sourceLineNo">323</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.323"></a>
-<span class="sourceLineNo">324</span> final long procId) throws Exception {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.325"></a>
-<span class="sourceLineNo">326</span> }<a name="line.326"></a>
-<span class="sourceLineNo">327</span><a name="line.327"></a>
-<span class="sourceLineNo">328</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.328"></a>
-<span class="sourceLineNo">329</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.329"></a>
-<span class="sourceLineNo">330</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.330"></a>
-<span class="sourceLineNo">331</span> }<a name="line.331"></a>
-<span class="sourceLineNo">332</span><a name="line.332"></a>
-<span class="sourceLineNo">333</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.333"></a>
-<span class="sourceLineNo">334</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.334"></a>
-<span class="sourceLineNo">335</span> throws Exception {<a name="line.335"></a>
-<span class="sourceLineNo">336</span> Procedure proc = procExec.getProcedure(procId);<a name="line.336"></a>
-<span class="sourceLineNo">337</span> waitProcedure(procExec, procId);<a name="line.337"></a>
-<span class="sourceLineNo">338</span> assertEquals(false, procExec.isRunning());<a name="line.338"></a>
-<span class="sourceLineNo">339</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.339"></a>
-<span class="sourceLineNo">340</span> proc = procExec.getProcedure(procId);<a name="line.340"></a>
-<span class="sourceLineNo">341</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.341"></a>
-<span class="sourceLineNo">342</span> if (customRestart != null) {<a name="line.342"></a>
-<span class="sourceLineNo">343</span> customRestart.run();<a name="line.343"></a>
-<span class="sourceLineNo">344</span> } else {<a name="line.344"></a>
-<span class="sourceLineNo">345</span> restart(procExec);<a name="line.345"></a>
-<span class="sourceLineNo">346</span> }<a name="line.346"></a>
-<span class="sourceLineNo">347</span> waitProcedure(procExec, procId);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> }<a name="line.348"></a>
-<span class="sourceLineNo">349</span><a name="line.349"></a>
-<span class="sourceLineNo">350</span> assertEquals(true, procExec.isRunning());<a name="line.350"></a>
-<span class="sourceLineNo">351</span> if (expectFailure) {<a name="line.351"></a>
-<span class="sourceLineNo">352</span> assertProcFailed(procExec, procId);<a name="line.352"></a>
-<span class="sourceLineNo">353</span> } else {<a name="line.353"></a>
-<span class="sourceLineNo">354</span> assertProcNotFailed(procExec, procId);<a name="line.354"></a>
+<span class="sourceLineNo">282</span> public static <TEnv> Throwable assertProcFailed(final ProcedureExecutor<TEnv> procExecutor,<a name="line.282"></a>
+<span class="sourceLineNo">283</span> final long procId) {<a name="line.283"></a>
+<span class="sourceLineNo">284</span> Procedure<?> result = procExecutor.getResult(procId);<a name="line.284"></a>
+<span class="sourceLineNo">285</span> assertTrue("expected procedure result", result != null);<a name="line.285"></a>
+<span class="sourceLineNo">286</span> return assertProcFailed(result);<a name="line.286"></a>
+<span class="sourceLineNo">287</span> }<a name="line.287"></a>
+<span class="sourceLineNo">288</span><a name="line.288"></a>
+<span class="sourceLineNo">289</span> public static Throwable assertProcFailed(final Procedure<?> result) {<a name="line.289"></a>
+<span class="sourceLineNo">290</span> assertEquals(true, result.isFailed());<a name="line.290"></a>
+<span class="sourceLineNo">291</span> LOG.info("procId=" + result.getProcId() + " exception: " + result.getException().getMessage());<a name="line.291"></a>
+<span class="sourceLineNo">292</span> return getExceptionCause(result);<a name="line.292"></a>
+<span class="sourceLineNo">293</span> }<a name="line.293"></a>
+<span class="sourceLineNo">294</span><a name="line.294"></a>
+<span class="sourceLineNo">295</span> public static void assertIsAbortException(final Procedure<?> result) {<a name="line.295"></a>
+<span class="sourceLineNo">296</span> Throwable cause = assertProcFailed(result);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> assertTrue("expected abort exception, got "+ cause, cause instanceof ProcedureAbortedException);<a name="line.297"></a>
+<span class="sourceLineNo">298</span> }<a name="line.298"></a>
+<span class="sourceLineNo">299</span><a name="line.299"></a>
+<span class="sourceLineNo">300</span> public static void assertIsTimeoutException(final Procedure<?> result) {<a name="line.300"></a>
+<span class="sourceLineNo">301</span> Throwable cause = assertProcFailed(result);<a name="line.301"></a>
+<span class="sourceLineNo">302</span> assertTrue("expected TimeoutIOException, got " + cause, cause instanceof TimeoutIOException);<a name="line.302"></a>
+<span class="sourceLineNo">303</span> }<a name="line.303"></a>
+<span class="sourceLineNo">304</span><a name="line.304"></a>
+<span class="sourceLineNo">305</span> public static void assertIsIllegalArgumentException(final Procedure<?> result) {<a name="line.305"></a>
+<span class="sourceLineNo">306</span> Throwable cause = assertProcFailed(result);<a name="line.306"></a>
+<span class="sourceLineNo">307</span> assertTrue("expected IllegalArgumentIOException, got " + cause,<a name="line.307"></a>
+<span class="sourceLineNo">308</span> cause instanceof IllegalArgumentIOException);<a name="line.308"></a>
+<span class="sourceLineNo">309</span> }<a name="line.309"></a>
+<span class="sourceLineNo">310</span><a name="line.310"></a>
+<span class="sourceLineNo">311</span> public static Throwable getExceptionCause(final Procedure<?> procInfo) {<a name="line.311"></a>
+<span class="sourceLineNo">312</span> assert procInfo.isFailed();<a name="line.312"></a>
+<span class="sourceLineNo">313</span> Throwable cause = procInfo.getException().getCause();<a name="line.313"></a>
+<span class="sourceLineNo">314</span> return cause == null ? procInfo.getException() : cause;<a name="line.314"></a>
+<span class="sourceLineNo">315</span> }<a name="line.315"></a>
+<span class="sourceLineNo">316</span><a name="line.316"></a>
+<span class="sourceLineNo">317</span> /**<a name="line.317"></a>
+<span class="sourceLineNo">318</span> * Run through all procedure flow states TWICE while also restarting<a name="line.318"></a>
+<span class="sourceLineNo">319</span> * procedure executor at each step; i.e force a reread of procedure store.<a name="line.319"></a>
+<span class="sourceLineNo">320</span> *<a name="line.320"></a>
+<span class="sourceLineNo">321</span> *<p>It does<a name="line.321"></a>
+<span class="sourceLineNo">322</span> * <ol><li>Execute step N - kill the executor before store update<a name="line.322"></a>
+<span class="sourceLineNo">323</span> * <li>Restart executor/store<a name="line.323"></a>
+<span class="sourceLineNo">324</span> * <li>Execute step N - and then save to store<a name="line.324"></a>
+<span class="sourceLineNo">325</span> * </ol><a name="line.325"></a>
+<span class="sourceLineNo">326</span> *<a name="line.326"></a>
+<span class="sourceLineNo">327</span> *<p>This is a good test for finding state that needs persisting and steps that are not<a name="line.327"></a>
+<span class="sourceLineNo">328</span> * idempotent.<a name="line.328"></a>
+<span class="sourceLineNo">329</span> */<a name="line.329"></a>
+<span class="sourceLineNo">330</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.330"></a>
+<span class="sourceLineNo">331</span> final long procId) throws Exception {<a name="line.331"></a>
+<span class="sourceLineNo">332</span> testRecoveryAndDoubleExecution(procExec, procId, false);<a name="line.332"></a>
+<span class="sourceLineNo">333</span> }<a name="line.333"></a>
+<span class="sourceLineNo">334</span><a name="line.334"></a>
+<span class="sourceLineNo">335</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.335"></a>
+<span class="sourceLineNo">336</span> final long procId, final boolean expectFailure) throws Exception {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> testRecoveryAndDoubleExecution(procExec, procId, expectFailure, null);<a name="line.337"></a>
+<span class="sourceLineNo">338</span> }<a name="line.338"></a>
+<span class="sourceLineNo">339</span><a name="line.339"></a>
+<span class="sourceLineNo">340</span> public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec,<a name="line.340"></a>
+<span class="sourceLineNo">341</span> final long procId, final boolean expectFailure, final Runnable customRestart)<a name="line.341"></a>
+<span class="sourceLineNo">342</span> throws Exception {<a name="line.342"></a>
+<span class="sourceLineNo">343</span> Procedure proc = procExec.getProcedure(procId);<a name="line.343"></a>
+<span class="sourceLineNo">344</span> waitProcedure(procExec, procId);<a name="line.344"></a>
+<span class="sourceLineNo">345</span> assertEquals(false, procExec.isRunning());<a name="line.345"></a>
+<span class="sourceLineNo">346</span> for (int i = 0; !procExec.isFinished(procId); ++i) {<a name="line.346"></a>
+<span class="sourceLineNo">347</span> proc = procExec.getProcedure(procId);<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.info("Restart " + i + " exec state: " + proc);<a name="line.348"></a>
+<span class="sourceLineNo">349</span> if (customRestart != null) {<a name="line.349"></a>
+<span class="sourceLineNo">350</span> customRestart.run();<a name="line.350"></a>
+<span class="sourceLineNo">351</span> } else {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> restart(procExec);<a name="line.352"></a>
+<span class="sourceLineNo">353</span> }<a name="line.353"></a>
+<span class="sourceLineNo">354</span> waitProcedure(procExec, procId);<a name="line.354"></a>
<span class="sourceLineNo">355</span> }<a name="line.355"></a>
-<span class="sourceLineNo">356</span> }<a name="line.356"></a>
-<span class="sourceLineNo">357</span><a name="line.357"></a>
-<span class="sourceLineNo">358</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.358"></a>
-<span class="sourceLineNo">359</span> public NoopProcedure() {}<a name="line.359"></a>
-<span class="sourceLineNo">360</span><a name="line.360"></a>
-<span class="sourceLineNo">361</span> @Override<a name="line.361"></a>
-<span class="sourceLineNo">362</span> protected Procedure[] execute(TEnv env)<a name="line.362"></a>
-<span class="sourceLineNo">363</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> return null;<a name="line.364"></a>
-<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span> @Override<a name="line.367"></a>
-<span class="sourceLineNo">368</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> @Override<a name="line.371"></a>
-<span class="sourceLineNo">372</span> protected boolean abort(TEnv env) { return false; }<a name="line.372"></a>
+<span class="sourceLineNo">356</span><a name="line.356"></a>
+<span class="sourceLineNo">357</span> assertEquals(true, procExec.isRunning());<a name="line.357"></a>
+<span class="sourceLineNo">358</span> if (expectFailure) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> assertProcFailed(procExec, procId);<a name="line.359"></a>
+<span class="sourceLineNo">360</span> } else {<a name="line.360"></a>
+<span class="sourceLineNo">361</span> assertProcNotFailed(procExec, procId);<a name="line.361"></a>
+<span class="sourceLineNo">362</span> }<a name="line.362"></a>
+<span class="sourceLineNo">363</span> }<a name="line.363"></a>
+<span class="sourceLineNo">364</span><a name="line.364"></a>
+<span class="sourceLineNo">365</span> public static class NoopProcedure<TEnv> extends Procedure<TEnv> {<a name="line.365"></a>
+<span class="sourceLineNo">366</span> public NoopProcedure() {}<a name="line.366"></a>
+<span class="sourceLineNo">367</span><a name="line.367"></a>
+<span class="sourceLineNo">368</span> @Override<a name="line.368"></a>
+<span class="sourceLineNo">369</span> protected Procedure[] execute(TEnv env)<a name="line.369"></a>
+<span class="sourceLineNo">370</span> throws ProcedureYieldException, ProcedureSuspendedException, InterruptedException {<a name="line.370"></a>
+<span class="sourceLineNo">371</span> return null;<a name="line.371"></a>
+<span class="sourceLineNo">372</span> }<a name="line.372"></a>
<span class="sourceLineNo">373</span><a name="line.373"></a>
<span class="sourceLineNo">374</span> @Override<a name="line.374"></a>
-<span class="sourceLineNo">375</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.375"></a>
-<span class="sourceLineNo">376</span> throws IOException {<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> @Override<a name="line.379"></a>
-<span class="sourceLineNo">380</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.380"></a>
-<span class="sourceLineNo">381</span> throws IOException {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> }<a name="line.382"></a>
-<span class="sourceLineNo">383</span> }<a name="line.383"></a>
-<span class="sourceLineNo">384</span><a name="line.384"></a>
-<span class="sourceLineNo">385</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.385"></a>
-<span class="sourceLineNo">386</span> private byte[] data = null;<a name="line.386"></a>
-<span class="sourceLineNo">387</span><a name="line.387"></a>
-<span class="sourceLineNo">388</span> public TestProcedure() {}<a name="line.388"></a>
-<span class="sourceLineNo">389</span><a name="line.389"></a>
-<span class="sourceLineNo">390</span> public TestProcedure(long procId) {<a name="line.390"></a>
-<span class="sourceLineNo">391</span> this(procId, 0);<a name="line.391"></a>
-<span class="sourceLineNo">392</span> }<a name="line.392"></a>
-<span class="sourceLineNo">393</span><a name="line.393"></a>
-<span class="sourceLineNo">394</span> public TestProcedure(long procId, long parentId) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span> this(procId, parentId, null);<a name="line.395"></a>
-<span class="sourceLineNo">396</span> }<a name="line.396"></a>
-<span class="sourceLineNo">397</span><a name="line.397"></a>
-<span class="sourceLineNo">398</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.398"></a>
-<span class="sourceLineNo">399</span> this(procId, parentId, parentId, data);<a name="line.399"></a>
-<span class="sourceLineNo">400</span> }<a name="line.400"></a>
-<span class="sourceLineNo">401</span><a name="line.401"></a>
-<span class="sourceLineNo">402</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> setData(data);<a name="line.403"></a>
-<span class="sourceLineNo">404</span> setProcId(procId);<a name="line.404"></a>
-<span class="sourceLineNo">405</span> if (parentId > 0) {<a name="line.405"></a>
-<span class="sourceLineNo">406</span> setParentProcId(parentId);<a name="line.406"></a>
-<span class="sourceLineNo">407</span> }<a name="line.407"></a>
-<span class="sourceLineNo">408</span> if (rootId > 0 || parentId > 0) {<a name="line.408"></a>
-<span class="sourceLineNo">409</span> setRootProcId(rootId);<a name="line.409"></a>
-<span class="sourceLineNo">410</span> }<a name="line.410"></a>
-<span class="sourceLineNo">411</span> }<a name="line.411"></a>
-<span class="sourceLineNo">412</span><a name="line.412"></a>
-<span class="sourceLineNo">413</span> public void addStackId(final int index) {<a name="line.413"></a>
-<span class="sourceLineNo">414</span> addStackIndex(index);<a name="line.414"></a>
-<span class="sourceLineNo">415</span> }<a name="line.415"></a>
-<span class="sourceLineNo">416</span><a name="line.416"></a>
-<span class="sourceLineNo">417</span> public void setSuccessState() {<a name="line.417"></a>
-<span class="sourceLineNo">418</span> setState(ProcedureState.SUCCESS);<a name="line.418"></a>
-<span class="sourceLineNo">419</span> }<a name="line.419"></a>
-<span class="sourceLineNo">420</span><a name="line.420"></a>
-<span class="sourceLineNo">421</span> public void setData(final byte[] data) {<a name="line.421"></a>
-<span class="sourceLineNo">422</span> this.data = data;<a name="line.422"></a>
-<span class="sourceLineNo">423</span> }<a name="line.423"></a>
-<span class="sourceLineNo">424</span><a name="line.424"></a>
-<span class="sourceLineNo">425</span> @Override<a name="line.425"></a>
-<span class="sourceLineNo">426</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.426"></a>
-<span class="sourceLineNo">427</span> throws IOException {<a name="line.427"></a>
-<span class="sourceLineNo">428</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.428"></a>
-<span class="sourceLineNo">429</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.429"></a>
-<span class="sourceLineNo">430</span> serializer.serialize(builder.build());<a name="line.430"></a>
-<span class="sourceLineNo">431</span> }<a name="line.431"></a>
-<span class="sourceLineNo">432</span><a name="line.432"></a>
-<span class="sourceLineNo">433</span> @Override<a name="line.433"></a>
-<span class="sourceLineNo">434</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.434"></a>
-<span class="sourceLineNo">435</span> throws IOException {<a name="line.435"></a>
-<span class="sourceLineNo">436</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.436"></a>
-<span class="sourceLineNo">437</span> ByteString dataString = bytesValue.getValue();<a name="line.437"></a>
-<span class="sourceLineNo">438</span><a name="line.438"></a>
-<span class="sourceLineNo">439</span> if (dataString.isEmpty()) {<a name="line.439"></a>
-<span class="sourceLineNo">440</span> data = null;<a name="line.440"></a>
-<span class="sourceLineNo">441</span> } else {<a name="line.441"></a>
-<span class="sourceLineNo">442</span> data = dataString.toByteArray();<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span> }<a name="line.444"></a>
+<span class="sourceLineNo">375</span> protected void rollback(TEnv env) throws IOException, InterruptedException {<a name="line.375"></a>
+<span class="sourceLineNo">376</span> }<a name="line.376"></a>
+<span class="sourceLineNo">377</span><a name="line.377"></a>
+<span class="sourceLineNo">378</span> @Override<a name="line.378"></a>
+<span class="sourceLineNo">379</span> protected boolean abort(TEnv env) { return false; }<a name="line.379"></a>
+<span class="sourceLineNo">380</span><a name="line.380"></a>
+<span class="sourceLineNo">381</span> @Override<a name="line.381"></a>
+<span class="sourceLineNo">382</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.382"></a>
+<span class="sourceLineNo">383</span> throws IOException {<a name="line.383"></a>
+<span class="sourceLineNo">384</span> }<a name="line.384"></a>
+<span class="sourceLineNo">385</span><a name="line.385"></a>
+<span class="sourceLineNo">386</span> @Override<a name="line.386"></a>
+<span class="sourceLineNo">387</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.387"></a>
+<span class="sourceLineNo">388</span> throws IOException {<a name="line.388"></a>
+<span class="sourceLineNo">389</span> }<a name="line.389"></a>
+<span class="sourceLineNo">390</span> }<a name="line.390"></a>
+<span class="sourceLineNo">391</span><a name="line.391"></a>
+<span class="sourceLineNo">392</span> public static class TestProcedure extends NoopProcedure<Void> {<a name="line.392"></a>
+<span class="sourceLineNo">393</span> private byte[] data = null;<a name="line.393"></a>
+<span class="sourceLineNo">394</span><a name="line.394"></a>
+<span class="sourceLineNo">395</span> public TestProcedure() {}<a name="line.395"></a>
+<span class="sourceLineNo">396</span><a name="line.396"></a>
+<span class="sourceLineNo">397</span> public TestProcedure(long procId) {<a name="line.397"></a>
+<span class="sourceLineNo">398</span> this(procId, 0);<a name="line.398"></a>
+<span class="sourceLineNo">399</span> }<a name="line.399"></a>
+<span class="sourceLineNo">400</span><a name="line.400"></a>
+<span class="sourceLineNo">401</span> public TestProcedure(long procId, long parentId) {<a name="line.401"></a>
+<span class="sourceLineNo">402</span> this(procId, parentId, null);<a name="line.402"></a>
+<span class="sourceLineNo">403</span> }<a name="line.403"></a>
+<span class="sourceLineNo">404</span><a name="line.404"></a>
+<span class="sourceLineNo">405</span> public TestProcedure(long procId, long parentId, byte[] data) {<a name="line.405"></a>
+<span class="sourceLineNo">406</span> this(procId, parentId, parentId, data);<a name="line.406"></a>
+<span class="sourceLineNo">407</span> }<a name="line.407"></a>
+<span class="sourceLineNo">408</span><a name="line.408"></a>
+<span class="sourceLineNo">409</span> public TestProcedure(long procId, long parentId, long rootId, byte[] data) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> setData(data);<a name="line.410"></a>
+<span class="sourceLineNo">411</span> setProcId(procId);<a name="line.411"></a>
+<span class="sourceLineNo">412</span> if (parentId > 0) {<a name="line.412"></a>
+<span class="sourceLineNo">413</span> setParentProcId(parentId);<a name="line.413"></a>
+<span class="sourceLineNo">414</span> }<a name="line.414"></a>
+<span class="sourceLineNo">415</span> if (rootId > 0 || parentId > 0) {<a name="line.415"></a>
+<span class="sourceLineNo">416</span> setRootProcId(rootId);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span> }<a name="line.418"></a>
+<span class="sourceLineNo">419</span><a name="line.419"></a>
+<span class="sourceLineNo">420</span> public void addStackId(final int index) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> addStackIndex(index);<a name="line.421"></a>
+<span class="sourceLineNo">422</span> }<a name="line.422"></a>
+<span class="sourceLineNo">423</span><a name="line.423"></a>
+<span class="sourceLineNo">424</span> public void setSuccessState() {<a name="line.424"></a>
+<span class="sourceLineNo">425</span> setState(ProcedureState.SUCCESS);<a name="line.425"></a>
+<span class="sourceLineNo">426</span> }<a name="line.426"></a>
+<span class="sourceLineNo">427</span><a name="line.427"></a>
+<span class="sourceLineNo">428</span> public void setData(final byte[] data) {<a name="line.428"></a>
+<span class="sourceLineNo">429</span> this.data = data;<a name="line.429"></a>
+<span class="sourceLineNo">430</span> }<a name="line.430"></a>
+<span class="sourceLineNo">431</span><a name="line.431"></a>
+<span class="sourceLineNo">432</span> @Override<a name="line.432"></a>
+<span class="sourceLineNo">433</span> protected void serializeStateData(ProcedureStateSerializer serializer)<a name="line.433"></a>
+<span class="sourceLineNo">434</span> throws IOException {<a name="line.434"></a>
+<span class="sourceLineNo">435</span> ByteString dataString = ByteString.copyFrom((data == null) ? new byte[0] : data);<a name="line.435"></a>
+<span class="sourceLineNo">436</span> BytesValue.Builder builder = BytesValue.newBuilder().setValue(dataString);<a name="line.436"></a>
+<span class="sourceLineNo">437</span> serializer.serialize(builder.build());<a name="line.437"></a>
+<span class="sourceLineNo">438</span> }<a name="line.438"></a>
+<span class="sourceLineNo">439</span><a name="line.439"></a>
+<span class="sourceLineNo">440</span> @Override<a name="line.440"></a>
+<span class="sourceLineNo">441</span> protected void deserializeStateData(ProcedureStateSerializer serializer)<a name="line.441"></a>
+<span class="sourceLineNo">442</span> throws IOException {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> BytesValue bytesValue = serializer.deserialize(BytesValue.class);<a name="line.443"></a>
+<span class="sourceLineNo">444</span> ByteString dataString = bytesValue.getValue();<a name="line.444"></a>
<span class="sourceLineNo">445</span><a name="line.445"></a>
-<span class="sourceLineNo">446</span> // Mark acquire/release lock functions public for test uses.<a name="line.446"></a>
-<span class="sourceLineNo">447</span> @Override<a name="line.447"></a>
-<span class="sourceLineNo">448</span> public LockState acquireLock(Void env) {<a name="line.448"></a>
-<span class="sourceLineNo">449</span> return LockState.LOCK_ACQUIRED;<a name="line.449"></a>
-<span class="sourceLineNo">450</span> }<a name="line.450"></a>
-<span class="sourceLineNo">451</span><a name="line.451"></a>
-<span class="sourceLineNo">452</span> @Override<a name="line.452"></a>
-<span class="sourceLineNo">453</span> public void releaseLock(Void env) {<a name="line.453"></a>
-<span class="sourceLineNo">454</span> // no-op<a name="line.454"></a>
-<span class="sourceLineNo">455</span> }<a name="line.455"></a>
-<span class="sourceLineNo">456</span> }<a name="line.456"></a>
-<span class="sourceLineNo">457</span><a name="line.457"></a>
-<span class="sourceLineNo">458</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.458"></a>
-<span class="sourceLineNo">459</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.459"></a>
-<span class="sourceLineNo">460</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.460"></a>
-<span class="sourceLineNo">461</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.461"></a>
-<span class="sourceLineNo">462</span><a name="line.462"></a>
-<span class="sourceLineNo">463</span> private Set<Long> procIds;<a name="line.463"></a>
-<span class="sourceLineNo">464</span> private long maxProcId = 0;<a name="line.464"></a>
-<span class="sourceLineNo">465</span><a name="line.465"></a>
-<span class="sourceLineNo">466</span> public LoadCounter() {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> this(null);<a name="line.467"></a>
-<span class="sourceLineNo">468</span> }<a name="line.468"></a>
+<span class="sourceLineNo">446</span> if (dataString.isEmpty()) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> data = null;<a name="line.447"></a>
+<span class="sourceLineNo">448</span> } else {<a name="line.448"></a>
+<span class="sourceLineNo">449</span> data = dataString.toByteArray();<a name="line.449"></a>
+<span class="sourceLineNo">450</span> }<a name="line.450"></a>
+<span class="sourceLineNo">451</span> }<a name="line.451"></a>
+<span class="sourceLineNo">452</span><a name="line.452"></a>
+<span class="sourceLineNo">453</span> // Mark acquire/release lock functions public for test uses.<a name="line.453"></a>
+<span class="sourceLineNo">454</span> @Override<a name="line.454"></a>
+<span class="sourceLineNo">455</span> public LockState acquireLock(Void env) {<a name="line.455"></a>
+<span class="sourceLineNo">456</span> return LockState.LOCK_ACQUIRED;<a name="line.456"></a>
+<span class="sourceLineNo">457</span> }<a name="line.457"></a>
+<span class="sourceLineNo">458</span><a name="line.458"></a>
+<span class="sourceLineNo">459</span> @Override<a name="line.459"></a>
+<span class="sourceLineNo">460</span> public void releaseLock(Void env) {<a name="line.460"></a>
+<span class="sourceLineNo">461</span> // no-op<a name="line.461"></a>
+<span class="sourceLineNo">462</span> }<a name="line.462"></a>
+<span class="sourceLineNo">463</span> }<a name="line.463"></a>
+<span class="sourceLineNo">464</span><a name="line.464"></a>
+<span class="sourceLineNo">465</span> public static class LoadCounter implements ProcedureStore.ProcedureLoader {<a name="line.465"></a>
+<span class="sourceLineNo">466</span> private final ArrayList<Procedure> corrupted = new ArrayList<>();<a name="line.466"></a>
+<span class="sourceLineNo">467</span> private final ArrayList<Procedure> completed = new ArrayList<>();<a name="line.467"></a>
+<span class="sourceLineNo">468</span> private final ArrayList<Procedure> runnable = new ArrayList<>();<a name="line.468"></a>
<span class="sourceLineNo">469</span><a name="line.469"></a>
-<span class="sourceLineNo">470</span> public LoadCounter(final Set<Long> procIds) {<a name="line.470"></a>
-<span class="sourceLineNo">471</span> this.procIds = procIds;<a name="line.471"></a>
-<span class="sourceLineNo">472</span> }<a name="line.472"></a>
-<span class="sourceLineNo">473</span><a name="line.473"></a>
-<span class="sourceLineNo">474</span> public void reset() {<a name="line.474"></a>
-<span class="sourceLineNo">475</span> reset(null);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> }<a name="line.476"></a>
-<span class="sourceLineNo">477</span><a name="line.477"></a>
-<span class="sourceLineNo">478</span> public void reset(final Set<Long> procIds) {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> corrupted.clear();<a name="line.479"></a>
-<span class="sourceLineNo">480</span> completed.clear();<a name="line.480"></a>
-<span class="sourceLineNo">481</span> runnable.clear();<a name="line.481"></a>
-<span class="sourceLineNo">482</span> this.procIds = procIds;<a name="line.482"></a>
-<span class="sourceLineNo">483</span> this.maxProcId = 0;<a name="line.483"></a>
-<span class="sourceLineNo">484</span> }<a name="line.484"></a>
-<span class="sourceLineNo">485</span><a name="line.485"></a>
-<span class="sourceLineNo">486</span> public long getMaxProcId() {<a name="line.486"></a>
-<span class="sourceLineNo">487</span> return maxProcId;<a name="line.487"></a>
-<span class="sourceLineNo">488</span> }<a name="line.488"></a>
-<span class="sourceLineNo">489</span><a name="line.489"></a>
-<span class="sourceLineNo">490</span> public ArrayList<Procedure> getRunnables() {<a name="line.490"></a>
-<span class="sourceLineNo">491</span> return runnable;<a name="line.491"></a>
-<span class="sourceLineNo">492</span> }<a name="line.492"></a>
-<span class="sourceLineNo">493</span><a name="line.493"></a>
-<span class="sourceLineNo">494</span> public int getRunnableCount() {<a name="line.494"></a>
-<span class="sourceLineNo">495</span> return runnable.size();<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> public ArrayList<Procedure> getCompleted() {<a name="line.498"></a>
-<span class="sourceLineNo">499</span> return completed;<a name="line.499"></a>
-<span class="sourceLineNo">500</span> }<a name="line.500"></a>
-<span class="sourceLineNo">501</span><a name="line.501"></a>
-<span class="sourceLineNo">502</span> public int getCompletedCount() {<a name="line.502"></a>
-<span class="sourceLineNo">503</span> return completed.size();<a name="line.503"></a>
-<span class="sourceLineNo">504</span> }<a name="line.504"></a>
-<span class="sourceLineNo">505</span><a name="line.505"></a>
-<span class="sourceLineNo">506</span> public int getLoadedCount() {<a name="line.506"></a>
-<span class="sourceLineNo">507</span> return runnable.size() + completed.size();<a name="line.507"></a>
-<span class="sourceLineNo">508</span> }<a name="line.508"></a>
-<span class="sourceLineNo">509</span><a name="line.509"></a>
-<span class="sourceLineNo">510</span> public ArrayList<Procedure> getCorrupted() {<a name="line.510"></a>
-<span class="sourceLineNo">511</span> return corrupted;<a name="line.511"></a>
-<span class="sourceLineNo">512</span> }<a name="line.512"></a>
-<span class="sourceLineNo">513</span><a name="line.513"></a>
-<span class="sourceLineNo">514</span> public int getCorruptedCount() {<a name="line.514"></a>
-<span class="sourceLineNo">515</span> return corrupted.size();<a name="line.515"></a>
-<span class="sourceLineNo">516</span> }<a name="line.516"></a>
-<span class="sourceLineNo">517</span><a name="line.517"></a>
-<span class="sourceLineNo">518</span> public boolean isRunnable(final long procId) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span> for (Procedure proc: runnable) {<a name="line.519"></a>
-<span class="sourceLineNo">520</span> if (proc.getProcId() == procId) {<a name="line.520"></a>
-<span class="sourceLineNo">521</span> return true;<a name="line.521"></a>
-<span class="sourceLineNo">522</span> }<a name="line.522"></a>
-<span class="sourceLineNo">523</span> }<a name="line.523"></a>
-<span class="sourceLineNo">524</span> return false;<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span><a name="line.526"></a>
-<span class="sourceLineNo">527</span> @Override<a name="line.527"></a>
-<span class="sourceLineNo">528</span> public void setMaxProcId(long maxProcId) {<a name="line.528"></a>
-<span class="sourceLineNo">529</span> this.maxProcId = maxProcId;<a name="line.529"></a>
-<span class="sourceLineNo">530</span> }<a name="line.530"></a>
-<span class="sourceLineNo">531</span><a name="line.531"></a>
-<span class="sourceLineNo">532</span> @Override<a name="line.532"></a>
-<span class="sourceLineNo">533</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.533"></a>
-<span class="sourceLineNo">534</span> while (procIter.hasNext()) {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> long procId;<a name="line.535"></a>
-<span class="sourceLineNo">536</span> if (procIter.isNextFinished()) {<a name="line.536"></a>
-<span class="sourceLineNo">537</span> Procedure<?> proc = procIter.next();<a name="line.537"></a>
-<span class="sourceLineNo">538</span> procId = proc.getProcId();<a name="line.538"></a>
-<span class="sourceLineNo">539</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.539"></a>
-<span class="sourceLineNo">540</span> completed.add(proc);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> } else {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> Procedure proc = procIter.next();<a name="line.542"></a>
-<span class="sourceLineNo">543</span> procId = proc.getProcId();<a name="line.543"></a>
-<span class="sourceLineNo">544</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.544"></a>
-<span class="sourceLineNo">545</span> runnable.add(proc);<a name="line.545"></a>
-<span class="sourceLineNo">546</span> }<a name="line.546"></a>
-<span class="sourceLineNo">547</span> if (procIds != null) {<a name="line.547"></a>
-<span class="sourceLineNo">548</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.548"></a>
-<span class="sourceLineNo">549</span> }<a name="line.549"></a>
-<span class="sourceLineNo">550</span> }<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> @Override<a name="line.553"></a>
-<span class="sourceLineNo">554</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.554"></a>
-<span class="sourceLineNo">555</span> while (procIter.hasNext()) {<a name="line.555"></a>
-<span class="sourceLineNo">556</span> Procedure proc = procIter.next();<a name="line.556"></a>
-<span class="sourceLineNo">557</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.557"></a>
-<span class="sourceLineNo">558</span> corrupted.add(proc);<a name="line.558"></a>
-<span class="sourceLineNo">559</span> }<a name="line.559"></a>
-<span class="sourceLineNo">560</span> }<a name="line.560"></a>
-<span class="sourceLineNo">561</span> }<a name="line.561"></a>
-<span class="sourceLineNo">562</span>}<a name="line.562"></a>
+<span class="sourceLineNo">470</span> private Set<Long> procIds;<a name="line.470"></a>
+<span class="sourceLineNo">471</span> private long maxProcId = 0;<a name="line.471"></a>
+<span class="sourceLineNo">472</span><a name="line.472"></a>
+<span class="sourceLineNo">473</span> public LoadCounter() {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> this(null);<a name="line.474"></a>
+<span class="sourceLineNo">475</span> }<a name="line.475"></a>
+<span class="sourceLineNo">476</span><a name="line.476"></a>
+<span class="sourceLineNo">477</span> public LoadCounter(final Set<Long> procIds) {<a name="line.477"></a>
+<span class="sourceLineNo">478</span> this.procIds = procIds;<a name="line.478"></a>
+<span class="sourceLineNo">479</span> }<a name="line.479"></a>
+<span class="sourceLineNo">480</span><a name="line.480"></a>
+<span class="sourceLineNo">481</span> public void reset() {<a name="line.481"></a>
+<span class="sourceLineNo">482</span> reset(null);<a name="line.482"></a>
+<span class="sourceLineNo">483</span> }<a name="line.483"></a>
+<span class="sourceLineNo">484</span><a name="line.484"></a>
+<span class="sourceLineNo">485</span> public void reset(final Set<Long> procIds) {<a name="line.485"></a>
+<span class="sourceLineNo">486</span> corrupted.clear();<a name="line.486"></a>
+<span class="sourceLineNo">487</span> completed.clear();<a name="line.487"></a>
+<span class="sourceLineNo">488</span> runnable.clear();<a name="line.488"></a>
+<span class="sourceLineNo">489</span> this.procIds = procIds;<a name="line.489"></a>
+<span class="sourceLineNo">490</span> this.maxProcId = 0;<a name="line.490"></a>
+<span class="sourceLineNo">491</span> }<a name="line.491"></a>
+<span class="sourceLineNo">492</span><a name="line.492"></a>
+<span class="sourceLineNo">493</span> public long getMaxProcId() {<a name="line.493"></a>
+<span class="sourceLineNo">494</span> return maxProcId;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> }<a name="line.495"></a>
+<span class="sourceLineNo">496</span><a name="line.496"></a>
+<span class="sourceLineNo">497</span> public ArrayList<Procedure> getRunnables() {<a name="line.497"></a>
+<span class="sourceLineNo">498</span> return runnable;<a name="line.498"></a>
+<span class="sourceLineNo">499</span> }<a name="line.499"></a>
+<span class="sourceLineNo">500</span><a name="line.500"></a>
+<span class="sourceLineNo">501</span> public int getRunnableCount() {<a name="line.501"></a>
+<span class="sourceLineNo">502</span> return runnable.size();<a name="line.502"></a>
+<span class="sourceLineNo">503</span> }<a name="line.503"></a>
+<span class="sourceLineNo">504</span><a name="line.504"></a>
+<span class="sourceLineNo">505</span> public ArrayList<Procedure> getCompleted() {<a name="line.505"></a>
+<span class="sourceLineNo">506</span> return completed;<a name="line.506"></a>
+<span class="sourceLineNo">507</span> }<a name="line.507"></a>
+<span class="sourceLineNo">508</span><a name="line.508"></a>
+<span class="sourceLineNo">509</span> public int getCompletedCount() {<a name="line.509"></a>
+<span class="sourceLineNo">510</span> return completed.size();<a name="line.510"></a>
+<span class="sourceLineNo">511</span> }<a name="line.511"></a>
+<span class="sourceLineNo">512</span><a name="line.512"></a>
+<span class="sourceLineNo">513</span> public int getLoadedCount() {<a name="line.513"></a>
+<span class="sourceLineNo">514</span> return runnable.size() + completed.size();<a name="line.514"></a>
+<span class="sourceLineNo">515</span> }<a name="line.515"></a>
+<span class="sourceLineNo">516</span><a name="line.516"></a>
+<span class="sourceLineNo">517</span> public ArrayList<Procedure> getCorrupted() {<a name="line.517"></a>
+<span class="sourceLineNo">518</span> return corrupted;<a name="line.518"></a>
+<span class="sourceLineNo">519</span> }<a name="line.519"></a>
+<span class="sourceLineNo">520</span><a name="line.520"></a>
+<span class="sourceLineNo">521</span> public int getCorruptedCount() {<a name="line.521"></a>
+<span class="sourceLineNo">522</span> return corrupted.size();<a name="line.522"></a>
+<span class="sourceLineNo">523</span> }<a name="line.523"></a>
+<span class="sourceLineNo">524</span><a name="line.524"></a>
+<span class="sourceLineNo">525</span> public boolean isRunnable(final long procId) {<a name="line.525"></a>
+<span class="sourceLineNo">526</span> for (Procedure proc: runnable) {<a name="line.526"></a>
+<span class="sourceLineNo">527</span> if (proc.getProcId() == procId) {<a name="line.527"></a>
+<span class="sourceLineNo">528</span> return true;<a name="line.528"></a>
+<span class="sourceLineNo">529</span> }<a name="line.529"></a>
+<span class="sourceLineNo">530</span> }<a name="line.530"></a>
+<span class="sourceLineNo">531</span> return false;<a name="line.531"></a>
+<span class="sourceLineNo">532</span> }<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span> @Override<a name="line.534"></a>
+<span class="sourceLineNo">535</span> public void setMaxProcId(long maxProcId) {<a name="line.535"></a>
+<span class="sourceLineNo">536</span> this.maxProcId = maxProcId;<a name="line.536"></a>
+<span class="sourceLineNo">537</span> }<a name="line.537"></a>
+<span class="sourceLineNo">538</span><a name="line.538"></a>
+<span class="sourceLineNo">539</span> @Override<a name="line.539"></a>
+<span class="sourceLineNo">540</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.540"></a>
+<span class="sourceLineNo">541</span> while (procIter.hasNext()) {<a name="line.541"></a>
+<span class="sourceLineNo">542</span> long procId;<a name="line.542"></a>
+<span class="sourceLineNo">543</span> if (procIter.isNextFinished()) {<a name="line.543"></a>
+<span class="sourceLineNo">544</span> Procedure<?> proc = procIter.next();<a name="line.544"></a>
+<span class="sourceLineNo">545</span> procId = proc.getProcId();<a name="line.545"></a>
+<span class="sourceLineNo">546</span> LOG.debug("loading completed procId=" + procId + ": " + proc);<a name="line.546"></a>
+<span class="sourceLineNo">547</span> completed.add(proc);<a name="line.547"></a>
+<span class="sourceLineNo">548</span> } else {<a name="line.548"></a>
+<span class="sourceLineNo">549</span> Procedure proc = procIter.next();<a name="line.549"></a>
+<span class="sourceLineNo">550</span> procId = proc.getProcId();<a name="line.550"></a>
+<span class="sourceLineNo">551</span> LOG.debug("loading runnable procId=" + procId + ": " + proc);<a name="line.551"></a>
+<span class="sourceLineNo">552</span> runnable.add(proc);<a name="line.552"></a>
+<span class="sourceLineNo">553</span> }<a name="line.553"></a>
+<span class="sourceLineNo">554</span> if (procIds != null) {<a name="line.554"></a>
+<span class="sourceLineNo">555</span> assertTrue("procId=" + procId + " unexpected", procIds.contains(procId));<a name="line.555"></a>
+<span class="sourceLineNo">556</span> }<a name="line.556"></a>
+<span class="sourceLineNo">557</span> }<a name="line.557"></a>
+<span class="sourceLineNo">558</span> }<a name="line.558"></a>
+<span class="sourceLineNo">559</span><a name="line.559"></a>
+<span class="sourceLineNo">560</span> @Override<a name="line.560"></a>
+<span class="sourceLineNo">561</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.561"></a>
+<span class="sourceLineNo">562</span> while (procIter.hasNext()) {<a name="line.562"></a>
+<span class="sourceLineNo">563</span> Procedure proc = procIter.next();<a name="line.563"></a>
+<span class="sourceLineNo">564</span> LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc);<a name="line.564"></a>
+<span class="sourceLineNo">565</span> corrupted.add(proc);<a name="line.565"></a>
+<span class="sourceLineNo">566</span> }<a name="line.566"></a>
+<span class="sourceLineNo">567</span> }<a name="line.567"></a>
+<span class="sourceLineNo">568</span> }<a name="line.568"></a>
+<span class="sourceLineNo">569</span>}<a name="line.569"></a>