You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2020/02/09 14:45:25 UTC
[hbase-site] branch asf-site updated: Published site at
24823ecfc9ddc9bbb473657d2ccf73cbd971ee2b.
This is an automated email from the ASF dual-hosted git repository.
git-site-role pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hbase-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 88900e9 Published site at 24823ecfc9ddc9bbb473657d2ccf73cbd971ee2b.
88900e9 is described below
commit 88900e978e560c2cb05c26b11e431442ed2f4f3e
Author: jenkins <bu...@apache.org>
AuthorDate: Sun Feb 9 14:45:13 2020 +0000
Published site at 24823ecfc9ddc9bbb473657d2ccf73cbd971ee2b.
---
acid-semantics.html | 2 +-
apache_hbase_reference_guide.pdf | 4 +-
book.html | 2 +-
bulk-loads.html | 2 +-
checkstyle-aggregate.html | 4 +-
checkstyle.rss | 16 +-
coc.html | 2 +-
dependencies.html | 2 +-
dependency-convergence.html | 4 +-
dependency-info.html | 2 +-
dependency-management.html | 2 +-
devapidocs/constant-values.html | 4 +-
.../apache/hadoop/hbase/backup/package-tree.html | 2 +-
.../apache/hadoop/hbase/client/package-tree.html | 22 +-
.../hadoop/hbase/coprocessor/package-tree.html | 2 +-
.../apache/hadoop/hbase/filter/package-tree.html | 6 +-
.../hadoop/hbase/hbtop/terminal/package-tree.html | 2 +-
.../org/apache/hadoop/hbase/http/package-tree.html | 2 +-
.../apache/hadoop/hbase/io/hfile/package-tree.html | 6 +-
.../hadoop/hbase/mapreduce/package-tree.html | 2 +-
.../hbase/master/assignment/package-tree.html | 2 +-
.../apache/hadoop/hbase/master/package-tree.html | 4 +-
.../hbase/master/procedure/package-tree.html | 4 +-
.../hadoop/hbase/monitoring/package-tree.html | 2 +-
.../org/apache/hadoop/hbase/package-tree.html | 14 +-
.../ProcedureExecutor.KeepAliveWorkerThread.html | 6 +-
...rocedureExecutor.ProcedureExecutorListener.html | 8 +-
.../procedure2/ProcedureExecutor.Testing.html | 24 +-
.../ProcedureExecutor.WorkerMonitor.html | 32 +-
.../procedure2/ProcedureExecutor.WorkerThread.html | 20 +-
.../hadoop/hbase/procedure2/ProcedureExecutor.html | 224 +-
.../hadoop/hbase/procedure2/package-tree.html | 4 +-
.../apache/hadoop/hbase/quotas/package-tree.html | 6 +-
.../hadoop/hbase/regionserver/package-tree.html | 18 +-
.../regionserver/querymatcher/package-tree.html | 2 +-
.../hbase/regionserver/wal/package-tree.html | 2 +-
.../hadoop/hbase/security/access/package-tree.html | 6 +-
.../apache/hadoop/hbase/security/package-tree.html | 2 +-
.../apache/hadoop/hbase/thrift/package-tree.html | 2 +-
.../org/apache/hadoop/hbase/util/package-tree.html | 8 +-
.../org/apache/hadoop/hbase/wal/package-tree.html | 2 +-
.../src-html/org/apache/hadoop/hbase/Version.html | 4 +-
.../ProcedureExecutor.KeepAliveWorkerThread.html | 4175 ++++++++++----------
...rocedureExecutor.ProcedureExecutorListener.html | 4175 ++++++++++----------
.../procedure2/ProcedureExecutor.Testing.html | 4175 ++++++++++----------
.../ProcedureExecutor.WorkerMonitor.html | 4175 ++++++++++----------
.../procedure2/ProcedureExecutor.WorkerThread.html | 4175 ++++++++++----------
.../hadoop/hbase/procedure2/ProcedureExecutor.html | 4175 ++++++++++----------
downloads.html | 2 +-
export_control.html | 2 +-
index.html | 2 +-
issue-tracking.html | 2 +-
mail-lists.html | 2 +-
metrics.html | 2 +-
old_news.html | 2 +-
plugin-management.html | 2 +-
plugins.html | 2 +-
poweredbyhbase.html | 2 +-
project-info.html | 2 +-
project-reports.html | 2 +-
project-summary.html | 2 +-
pseudo-distributed.html | 2 +-
replication.html | 2 +-
resources.html | 2 +-
source-repository.html | 2 +-
sponsors.html | 2 +-
supportingprojects.html | 2 +-
team-list.html | 2 +-
testdevapidocs/allclasses-frame.html | 1 +
testdevapidocs/allclasses-noframe.html | 1 +
testdevapidocs/index-all.html | 16 +
.../hadoop/hbase/class-use/HBaseClassTestRule.html | 4 +
.../hbase/class-use/HBaseTestingUtility.html | 46 +-
.../hadoop/hbase/client/AbstractTestShell.html | 16 +-
.../apache/hadoop/hbase/io/hfile/package-tree.html | 2 +-
...oseRegionWhileRSCrash.DummyServerProcedure.html | 4 +-
.../TestDeadServerMetricRegionChore.html} | 146 +-
.../hbase/master/assignment/TestHbckChore.html | 4 +-
.../class-use/TestDeadServerMetricRegionChore.html | 125 +
.../hbase/master/assignment/package-frame.html | 1 +
.../hbase/master/assignment/package-summary.html | 106 +-
.../hbase/master/assignment/package-tree.html | 1 +
.../org/apache/hadoop/hbase/package-tree.html | 8 +-
.../hadoop/hbase/procedure/package-tree.html | 8 +-
.../hadoop/hbase/procedure2/package-tree.html | 2 +-
.../hadoop/hbase/regionserver/package-tree.html | 4 +-
.../org/apache/hadoop/hbase/test/package-tree.html | 4 +-
.../org/apache/hadoop/hbase/wal/package-tree.html | 2 +-
testdevapidocs/overview-tree.html | 1 +
.../hadoop/hbase/client/AbstractTestShell.html | 119 +-
.../TestDeadServerMetricRegionChore.html | 137 +
91 files changed, 13332 insertions(+), 13002 deletions(-)
diff --git a/acid-semantics.html b/acid-semantics.html
index f24ab0d..2779b37 100644
--- a/acid-semantics.html
+++ b/acid-semantics.html
@@ -467,7 +467,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/apache_hbase_reference_guide.pdf b/apache_hbase_reference_guide.pdf
index ba0f344..9b645e0 100644
--- a/apache_hbase_reference_guide.pdf
+++ b/apache_hbase_reference_guide.pdf
@@ -5,8 +5,8 @@
/Author (Apache HBase Team)
/Creator (Asciidoctor PDF 1.5.0.rc.2, based on Prawn 2.2.2)
/Producer (Apache HBase Team)
-/ModDate (D:20200208143029+00'00')
-/CreationDate (D:20200208144143+00'00')
+/ModDate (D:20200209143236+00'00')
+/CreationDate (D:20200209144234+00'00')
>>
endobj
2 0 obj
diff --git a/book.html b/book.html
index 2f4d984..d3df76e 100644
--- a/book.html
+++ b/book.html
@@ -44451,7 +44451,7 @@ org/apache/hadoop/hbase/security/access/AccessControlClient.revoke:(Lorg/apache/
<div id="footer">
<div id="footer-text">
Version 3.0.0-SNAPSHOT<br>
-Last updated 2020-02-08 14:30:29 UTC
+Last updated 2020-02-09 14:32:36 UTC
</div>
</div>
</body>
diff --git a/bulk-loads.html b/bulk-loads.html
index c06db0c..ebe29d0 100644
--- a/bulk-loads.html
+++ b/bulk-loads.html
@@ -172,7 +172,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/checkstyle-aggregate.html b/checkstyle-aggregate.html
index 0aa320f..fc1135a 100644
--- a/checkstyle-aggregate.html
+++ b/checkstyle-aggregate.html
@@ -166,7 +166,7 @@
<th><img src="images/icon_warning_sml.gif" alt="" /> Warnings</th>
<th><img src="images/icon_error_sml.gif" alt="" /> Errors</th></tr>
<tr class="b">
-<td>4187</td>
+<td>4188</td>
<td>0</td>
<td>0</td>
<td>10485</td></tr></table></div>
@@ -83740,7 +83740,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/checkstyle.rss b/checkstyle.rss
index 856e4cb..ddf6518 100644
--- a/checkstyle.rss
+++ b/checkstyle.rss
@@ -25,7 +25,7 @@ under the License.
<language>en-us</language>
<copyright>©2007 - 2020 The Apache Software Foundation</copyright>
<item>
- <title>File: 4187,
+ <title>File: 4188,
Errors: 10485,
Warnings: 0,
Infos: 0
@@ -8768,6 +8768,20 @@ under the License.
</tr>
<tr>
<td>
+ <a href="https://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.master.assignment.TestDeadServerMetricRegionChore.java">org/apache/hadoop/hbase/master/assignment/TestDeadServerMetricRegionChore.java</a>
+ </td>
+ <td>
+ 0
+ </td>
+ <td>
+ 0
+ </td>
+ <td>
+ 0
+ </td>
+ </tr>
+ <tr>
+ <td>
<a href="https://hbase.apache.org/checkstyle.html#org.apache.hadoop.hbase.procedure2.store.region.RegionFlusherAndCompactor.java">org/apache/hadoop/hbase/procedure2/store/region/RegionFlusherAndCompactor.java</a>
</td>
<td>
diff --git a/coc.html b/coc.html
index 7ceb983..e602bdb 100644
--- a/coc.html
+++ b/coc.html
@@ -241,7 +241,7 @@ email to <a class="externalLink" href="mailto:private@hbase.apache.org">the priv
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependencies.html b/dependencies.html
index 4d6923d..f3565b2 100644
--- a/dependencies.html
+++ b/dependencies.html
@@ -313,7 +313,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependency-convergence.html b/dependency-convergence.html
index 6242df6..1708f63 100644
--- a/dependency-convergence.html
+++ b/dependency-convergence.html
@@ -173,7 +173,7 @@
<td>297</td></tr>
<tr class="b">
<th>Number of unique artifacts (NOA):</th>
-<td>306</td></tr>
+<td>305</td></tr>
<tr class="a">
<th>Number of version-conflicting artifacts (NOC):</th>
<td>8</td></tr>
@@ -558,7 +558,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependency-info.html b/dependency-info.html
index 768acbf..fd9fe5f 100644
--- a/dependency-info.html
+++ b/dependency-info.html
@@ -194,7 +194,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/dependency-management.html b/dependency-management.html
index a96ccba..e42b6c8 100644
--- a/dependency-management.html
+++ b/dependency-management.html
@@ -928,7 +928,7 @@
<div class="row">
<p>Copyright ©2007–2020
<a href="https://www.apache.org/">The Apache Software Foundation</a>.
-All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-08</li>
+All rights reserved. <li id="publishDate" class="pull-right">Last Published: 2020-02-09</li>
</p>
</div>
<p id="poweredBy" class="pull-right"><a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy"><img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" /></a>
diff --git a/devapidocs/constant-values.html b/devapidocs/constant-values.html
index f8a9ecf..022406e 100644
--- a/devapidocs/constant-values.html
+++ b/devapidocs/constant-values.html
@@ -4109,14 +4109,14 @@
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#date">date</a></code></td>
-<td class="colLast"><code>"Sat Feb 8 14:38:19 UTC 2020"</code></td>
+<td class="colLast"><code>"Sun Feb 9 14:38:57 UTC 2020"</code></td>
</tr>
<tr class="rowColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.revision">
<!-- -->
</a><code>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a></code></td>
<td><code><a href="org/apache/hadoop/hbase/Version.html#revision">revision</a></code></td>
-<td class="colLast"><code>"546828685e2a7c2d686925a80c7a68dd4d4e3f8d"</code></td>
+<td class="colLast"><code>"24823ecfc9ddc9bbb473657d2ccf73cbd971ee2b"</code></td>
</tr>
<tr class="altColor">
<td class="colFirst"><a name="org.apache.hadoop.hbase.Version.srcChecksum">
diff --git a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
index 815bd73..2f3a624 100644
--- a/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/backup/package-tree.html
@@ -167,10 +167,10 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupRestoreConstants.BackupCommand.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupRestoreConstants.BackupCommand</span></a></li>
<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupState.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupInfo.BackupPhase.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupInfo.BackupPhase</span></a></li>
<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupType.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.backup.<a href="../../../../../org/apache/hadoop/hbase/backup/BackupRestoreConstants.BackupCommand.html" title="enum in org.apache.hadoop.hbase.backup"><span class="typeNameLink">BackupRestoreConstants.BackupCommand</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/client/package-tree.html b/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
index 5788797..4c6c47f 100644
--- a/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/client/package-tree.html
@@ -426,21 +426,21 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AbstractResponse.ResponseType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AbstractResponse.ResponseType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MasterSwitchType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MasterSwitchType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/TableState.State.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">TableState.State</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MobCompactPartitionPolicy</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/SnapshotType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">SnapshotType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/IsolationLevel.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">IsolationLevel</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Consistency.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Consistency</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Durability.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Durability</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanControllerState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactionState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactionState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RequestController.ReturnCode.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RequestController.ReturnCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Durability.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Durability</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/SnapshotType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">SnapshotType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AbstractResponse.ResponseType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AbstractResponse.ResponseType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Scan.ReadType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Scan.ReadType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MobCompactPartitionPolicy</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/TableState.State.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">TableState.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/MasterSwitchType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">MasterSwitchType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RegionLocateType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RegionLocateType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/RequestController.ReturnCode.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">RequestController.ReturnCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/Consistency.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">Consistency</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">AsyncScanSingleRegionRpcRetryingCaller.ScanResumerState</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.client.<a href="../../../../../org/apache/hadoop/hbase/client/CompactType.html" title="enum in org.apache.hadoop.hbase.client"><span class="typeNameLink">CompactType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html b/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
index 06e569c..675c699 100644
--- a/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/coprocessor/package-tree.html
@@ -201,8 +201,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/RegionObserver.MutationType.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">RegionObserver.MutationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/MetaTableMetrics.MetaTableOps.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">MetaTableMetrics.MetaTableOps</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.coprocessor.<a href="../../../../../org/apache/hadoop/hbase/coprocessor/RegionObserver.MutationType.html" title="enum in org.apache.hadoop.hbase.coprocessor"><span class="typeNameLink">RegionObserver.MutationType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html b/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
index bdc5808..9f7b870 100644
--- a/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/filter/package-tree.html
@@ -190,12 +190,12 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/RegexStringComparator.EngineType.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">RegexStringComparator.EngineType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.Order.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.Order</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/Filter.ReturnCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">Filter.ReturnCode</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterList.Operator.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterList.Operator</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.SatisfiesCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.SatisfiesCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/RegexStringComparator.EngineType.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">RegexStringComparator.EngineType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/BitComparator.BitwiseOp.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">BitComparator.BitwiseOp</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FuzzyRowFilter.SatisfiesCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FuzzyRowFilter.SatisfiesCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterList.Operator.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterList.Operator</span></a></li>
<li type="circle">org.apache.hadoop.hbase.filter.<a href="../../../../../org/apache/hadoop/hbase/filter/FilterWrapper.FilterRowRetCode.html" title="enum in org.apache.hadoop.hbase.filter"><span class="typeNameLink">FilterWrapper.FilterRowRetCode</span></a></li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/hbtop/terminal/package-tree.html b/devapidocs/org/apache/hadoop/hbase/hbtop/terminal/package-tree.html
index b8f3866..007e90f 100644
--- a/devapidocs/org/apache/hadoop/hbase/hbtop/terminal/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/hbtop/terminal/package-tree.html
@@ -107,8 +107,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.hbtop.terminal.<a href="../../../../../../org/apache/hadoop/hbase/hbtop/terminal/Color.html" title="enum in org.apache.hadoop.hbase.hbtop.terminal"><span class="typeNameLink">Color</span></a></li>
<li type="circle">org.apache.hadoop.hbase.hbtop.terminal.<a href="../../../../../../org/apache/hadoop/hbase/hbtop/terminal/KeyPress.Type.html" title="enum in org.apache.hadoop.hbase.hbtop.terminal"><span class="typeNameLink">KeyPress.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.hbtop.terminal.<a href="../../../../../../org/apache/hadoop/hbase/hbtop/terminal/Color.html" title="enum in org.apache.hadoop.hbase.hbtop.terminal"><span class="typeNameLink">Color</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/http/package-tree.html b/devapidocs/org/apache/hadoop/hbase/http/package-tree.html
index 54adced..136eaf6 100644
--- a/devapidocs/org/apache/hadoop/hbase/http/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/http/package-tree.html
@@ -140,9 +140,9 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.http.<a href="../../../../../org/apache/hadoop/hbase/http/ProfileServlet.Output.html" title="enum in org.apache.hadoop.hbase.http"><span class="typeNameLink">ProfileServlet.Output</span></a></li>
<li type="circle">org.apache.hadoop.hbase.http.<a href="../../../../../org/apache/hadoop/hbase/http/HttpConfig.Policy.html" title="enum in org.apache.hadoop.hbase.http"><span class="typeNameLink">HttpConfig.Policy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.http.<a href="../../../../../org/apache/hadoop/hbase/http/ProfileServlet.Event.html" title="enum in org.apache.hadoop.hbase.http"><span class="typeNameLink">ProfileServlet.Event</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.http.<a href="../../../../../org/apache/hadoop/hbase/http/ProfileServlet.Output.html" title="enum in org.apache.hadoop.hbase.http"><span class="typeNameLink">ProfileServlet.Output</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html b/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
index d53116c..8110dab 100644
--- a/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/io/hfile/package-tree.html
@@ -305,12 +305,12 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockPriority.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockPriority</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/ReaderContext.ReaderType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">ReaderContext.ReaderType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.ExternalBlockCaches.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockCacheFactory.ExternalBlockCaches</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/HFileBlock.Writer.State.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">HFileBlock.Writer.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockPriority.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockPriority</span></a></li>
<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.BlockCategory.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType.BlockCategory</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockType.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.io.hfile.<a href="../../../../../../org/apache/hadoop/hbase/io/hfile/BlockCacheFactory.ExternalBlockCaches.html" title="enum in org.apache.hadoop.hbase.io.hfile"><span class="typeNameLink">BlockCacheFactory.ExternalBlockCaches</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html b/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
index a069ef9..639a844 100644
--- a/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/mapreduce/package-tree.html
@@ -297,9 +297,9 @@
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/TableSplit.Version.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">TableSplit.Version</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/CellCounter.CellCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">CellCounter.CellCounterMapper.Counters</span></a></li>
<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/SyncTable.SyncMapper.Counter.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">SyncTable.SyncMapper.Counter</span></a></li>
<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/RowCounter.RowCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">RowCounter.RowCounterMapper.Counters</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.mapreduce.<a href="../../../../../org/apache/hadoop/hbase/mapreduce/CellCounter.CellCounterMapper.Counters.html" title="enum in org.apache.hadoop.hbase.mapreduce"><span class="typeNameLink">CellCounter.CellCounterMapper.Counters</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/master/assignment/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/assignment/package-tree.html
index f8add9f..5c89e3f 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/assignment/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/assignment/package-tree.html
@@ -151,8 +151,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.master.assignment.<a href="../../../../../../org/apache/hadoop/hbase/master/assignment/ServerState.html" title="enum in org.apache.hadoop.hbase.master.assignment"><span class="typeNameLink">ServerState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.assignment.<a href="../../../../../../org/apache/hadoop/hbase/master/assignment/TransitRegionStateProcedure.TransitionType.html" title="enum in org.apache.hadoop.hbase.master.assignment"><span class="typeNameLink">TransitRegionStateProcedure.TransitionType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.assignment.<a href="../../../../../../org/apache/hadoop/hbase/master/assignment/ServerState.html" title="enum in org.apache.hadoop.hbase.master.assignment"><span class="typeNameLink">ServerState</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/master/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
index a9d4e62..3e3fc7c 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/package-tree.html
@@ -360,10 +360,10 @@
<ul>
<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.ResubmitDirective.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.ResubmitDirective</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MetricsMasterSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MetricsMasterSourceFactoryImpl.FactoryStorage</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/RegionState.State.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">RegionState.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.TerminationStatus</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/MasterRpcServices.BalanceSwitchMode.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">MasterRpcServices.BalanceSwitchMode</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/ServerManager.ServerLiveState.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">ServerManager.ServerLiveState</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/SplitLogManager.TerminationStatus.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">SplitLogManager.TerminationStatus</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.<a href="../../../../../org/apache/hadoop/hbase/master/RegionState.State.html" title="enum in org.apache.hadoop.hbase.master"><span class="typeNameLink">RegionState.State</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html b/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
index 8038397..2878942 100644
--- a/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/master/procedure/package-tree.html
@@ -221,10 +221,10 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
+<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.ServerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">ServerProcedureInterface.ServerOperationType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.TableOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">TableProcedureInterface.TableOperationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/PeerProcedureInterface.PeerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">PeerProcedureInterface.PeerOperationType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/MetaProcedureInterface.MetaOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">MetaProcedureInterface.MetaOperationType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.TableOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">TableProcedureInterface.TableOperationType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.master.procedure.<a href="../../../../../../org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.ServerOperationType.html" title="enum in org.apache.hadoop.hbase.master.procedure"><span class="typeNameLink">ServerProcedureInterface.ServerOperationType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/monitoring/package-tree.html b/devapidocs/org/apache/hadoop/hbase/monitoring/package-tree.html
index 2b5bd29..3e65824 100644
--- a/devapidocs/org/apache/hadoop/hbase/monitoring/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/monitoring/package-tree.html
@@ -127,8 +127,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.monitoring.<a href="../../../../../org/apache/hadoop/hbase/monitoring/TaskMonitor.TaskFilter.TaskType.html" title="enum in org.apache.hadoop.hbase.monitoring"><span class="typeNameLink">TaskMonitor.TaskFilter.TaskType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.monitoring.<a href="../../../../../org/apache/hadoop/hbase/monitoring/MonitoredTask.State.html" title="enum in org.apache.hadoop.hbase.monitoring"><span class="typeNameLink">MonitoredTask.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.monitoring.<a href="../../../../../org/apache/hadoop/hbase/monitoring/TaskMonitor.TaskFilter.TaskType.html" title="enum in org.apache.hadoop.hbase.monitoring"><span class="typeNameLink">TaskMonitor.TaskFilter.TaskType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/package-tree.html b/devapidocs/org/apache/hadoop/hbase/package-tree.html
index 4a9191c..36887ee 100644
--- a/devapidocs/org/apache/hadoop/hbase/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/package-tree.html
@@ -431,18 +431,18 @@
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HealthChecker.HealthCheckerExitStatus.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HealthChecker.HealthCheckerExitStatus</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CellBuilderType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CellBuilderType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MemoryCompactionPolicy.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MemoryCompactionPolicy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Size.Unit.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Size.Unit</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HConstants.OperationStatusCode.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HConstants.OperationStatusCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ClusterMetrics.Option</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompareOperator.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompareOperator</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeyValue.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeyValue.Type</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/KeepDeletedCells.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">KeepDeletedCells</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompareOperator.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompareOperator</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CompatibilitySingletonFactory.SingletonStorage.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CompatibilitySingletonFactory.SingletonStorage</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MetaTableAccessor.QueryType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MetaTableAccessor.QueryType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Cell.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Cell.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/ClusterMetrics.Option.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">ClusterMetrics.Option</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/CellBuilderType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">CellBuilderType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Coprocessor.State.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Coprocessor.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/HConstants.OperationStatusCode.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">HConstants.OperationStatusCode</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/Cell.Type.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">Cell.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MemoryCompactionPolicy.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MemoryCompactionPolicy</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.<a href="../../../../org/apache/hadoop/hbase/MetaTableAccessor.QueryType.html" title="enum in org.apache.hadoop.hbase"><span class="typeNameLink">MetaTableAccessor.QueryType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
index 85cf011..0033dbb 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
@@ -132,7 +132,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.2024">ProcedureExecutor.KeepAliveWorkerThread</a>
+<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.2031">ProcedureExecutor.KeepAliveWorkerThread</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a></pre>
</li>
</ul>
@@ -252,7 +252,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExec
<ul class="blockListLast">
<li class="blockList">
<h4>KeepAliveWorkerThread</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.2025">KeepAliveWorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.2032">KeepAliveWorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
</li>
</ul>
</li>
@@ -269,7 +269,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExec
<ul class="blockListLast">
<li class="blockList">
<h4>keepAlive</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.2030">keepAlive</a>(long lastUpdate)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html#line.2037">keepAlive</a>(long lastUpdate)</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#keepAlive-long-">keepAlive</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a></code></dd>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
index b59bf87..8818c35 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
@@ -105,7 +105,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.156">ProcedureExecutor.ProcedureExecutorListener</a></pre>
+<pre>public static interface <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.157">ProcedureExecutor.ProcedureExecutorListener</a></pre>
</li>
</ul>
</div>
@@ -157,7 +157,7 @@ var activeTableTab = "activeTableTab";
<ul class="blockList">
<li class="blockList">
<h4>procedureLoaded</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.157">procedureLoaded</a>(long procId)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.158">procedureLoaded</a>(long procId)</pre>
</li>
</ul>
<a name="procedureAdded-long-">
@@ -166,7 +166,7 @@ var activeTableTab = "activeTableTab";
<ul class="blockList">
<li class="blockList">
<h4>procedureAdded</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.158">procedureAdded</a>(long procId)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.159">procedureAdded</a>(long procId)</pre>
</li>
</ul>
<a name="procedureFinished-long-">
@@ -175,7 +175,7 @@ var activeTableTab = "activeTableTab";
<ul class="blockListLast">
<li class="blockList">
<h4>procedureFinished</h4>
-<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.159">procedureFinished</a>(long procId)</pre>
+<pre>void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html#line.160">procedureFinished</a>(long procId)</pre>
</li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
index 565b90d..bcd9e09 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
@@ -113,7 +113,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.104">ProcedureExecutor.Testing</a>
+<pre>public static class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.105">ProcedureExecutor.Testing</a>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">Class with parameters describing how to fail/die when in testing-context.</div>
</li>
@@ -239,7 +239,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>killIfHasParent</h4>
-<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.105">killIfHasParent</a></pre>
+<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.106">killIfHasParent</a></pre>
</li>
</ul>
<a name="killIfSuspended">
@@ -248,7 +248,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>killIfSuspended</h4>
-<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.106">killIfSuspended</a></pre>
+<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.107">killIfSuspended</a></pre>
</li>
</ul>
<a name="killBeforeStoreUpdate">
@@ -257,7 +257,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>killBeforeStoreUpdate</h4>
-<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.112">killBeforeStoreUpdate</a></pre>
+<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.113">killBeforeStoreUpdate</a></pre>
<div class="block">Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is
persisting all the state it needs to recover after a crash.</div>
</li>
@@ -268,7 +268,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>toggleKillBeforeStoreUpdate</h4>
-<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.113">toggleKillBeforeStoreUpdate</a></pre>
+<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.114">toggleKillBeforeStoreUpdate</a></pre>
</li>
</ul>
<a name="killAfterStoreUpdate">
@@ -277,7 +277,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>killAfterStoreUpdate</h4>
-<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.120">killAfterStoreUpdate</a></pre>
+<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.121">killAfterStoreUpdate</a></pre>
<div class="block">Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978
is about a case where memory-state was being set after store to WAL where a crash could
cause us to get stuck. This flag allows killing at what was a vulnerable time.</div>
@@ -289,7 +289,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>toggleKillAfterStoreUpdate</h4>
-<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.121">toggleKillAfterStoreUpdate</a></pre>
+<pre>protected volatile boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.122">toggleKillAfterStoreUpdate</a></pre>
</li>
</ul>
</li>
@@ -306,7 +306,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>Testing</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.104">Testing</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.105">Testing</a>()</pre>
</li>
</ul>
</li>
@@ -323,7 +323,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>shouldKillBeforeStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.123">shouldKillBeforeStoreUpdate</a>()</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.124">shouldKillBeforeStoreUpdate</a>()</pre>
</li>
</ul>
<a name="shouldKillBeforeStoreUpdate-boolean-boolean-">
@@ -332,7 +332,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>shouldKillBeforeStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.132">shouldKillBeforeStoreUpdate</a>(boolean isSuspended,
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.133">shouldKillBeforeStoreUpdate</a>(boolean isSuspended,
boolean hasParent)</pre>
</li>
</ul>
@@ -342,7 +342,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>shouldKillAfterStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.142">shouldKillAfterStoreUpdate</a>()</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.143">shouldKillAfterStoreUpdate</a>()</pre>
</li>
</ul>
<a name="shouldKillAfterStoreUpdate-boolean-">
@@ -351,7 +351,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>shouldKillAfterStoreUpdate</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.151">shouldKillAfterStoreUpdate</a>(boolean isSuspended)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html#line.152">shouldKillAfterStoreUpdate</a>(boolean isSuspended)</pre>
</li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
index 6ad92cf..804d0dc 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html
@@ -127,7 +127,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.2043">ProcedureExecutor.WorkerMonitor</a>
+<pre>private final class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.2050">ProcedureExecutor.WorkerMonitor</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.html" title="class in org.apache.hadoop.hbase.procedure2">InlineChore</a></pre>
</li>
</ul>
@@ -277,7 +277,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>WORKER_MONITOR_INTERVAL_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2044">WORKER_MONITOR_INTERVAL_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2051">WORKER_MONITOR_INTERVAL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.WORKER_MONITOR_INTERVAL_CONF_KEY">Constant Field Values</a></dd>
@@ -290,7 +290,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_MONITOR_INTERVAL</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2046">DEFAULT_WORKER_MONITOR_INTERVAL</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2053">DEFAULT_WORKER_MONITOR_INTERVAL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.DEFAULT_WORKER_MONITOR_INTERVAL">Constant Field Values</a></dd>
@@ -303,7 +303,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>WORKER_STUCK_THRESHOLD_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2048">WORKER_STUCK_THRESHOLD_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2055">WORKER_STUCK_THRESHOLD_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.WORKER_STUCK_THRESHOLD_CONF_KEY">Constant Field Values</a></dd>
@@ -316,7 +316,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_STUCK_THRESHOLD</h4>
-<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2050">DEFAULT_WORKER_STUCK_THRESHOLD</a></pre>
+<pre>private static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2057">DEFAULT_WORKER_STUCK_THRESHOLD</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.DEFAULT_WORKER_STUCK_THRESHOLD">Constant Field Values</a></dd>
@@ -329,7 +329,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2052">WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2059">WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY">Constant Field Values</a></dd>
@@ -342,7 +342,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_ADD_STUCK_PERCENTAGE</h4>
-<pre>private static final float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2054">DEFAULT_WORKER_ADD_STUCK_PERCENTAGE</a></pre>
+<pre>private static final float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2061">DEFAULT_WORKER_ADD_STUCK_PERCENTAGE</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WorkerMonitor.DEFAULT_WORKER_ADD_STUCK_PERCENTAGE">Constant Field Values</a></dd>
@@ -355,7 +355,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>addWorkerStuckPercentage</h4>
-<pre>private float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2056">addWorkerStuckPercentage</a></pre>
+<pre>private float <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2063">addWorkerStuckPercentage</a></pre>
</li>
</ul>
<a name="timeoutInterval">
@@ -364,7 +364,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>timeoutInterval</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2057">timeoutInterval</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2064">timeoutInterval</a></pre>
</li>
</ul>
<a name="stuckThreshold">
@@ -373,7 +373,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockListLast">
<li class="blockList">
<h4>stuckThreshold</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2058">stuckThreshold</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2065">stuckThreshold</a></pre>
</li>
</ul>
</li>
@@ -390,7 +390,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockListLast">
<li class="blockList">
<h4>WorkerMonitor</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2060">WorkerMonitor</a>()</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2067">WorkerMonitor</a>()</pre>
</li>
</ul>
</li>
@@ -407,7 +407,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>run</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2065">run</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2072">run</a>()</pre>
</li>
</ul>
<a name="checkForStuckWorkers--">
@@ -416,7 +416,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>checkForStuckWorkers</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2073">checkForStuckWorkers</a>()</pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2080">checkForStuckWorkers</a>()</pre>
</li>
</ul>
<a name="checkThreadCount-int-">
@@ -425,7 +425,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>checkThreadCount</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2089">checkThreadCount</a>(int stuckCount)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2096">checkThreadCount</a>(int stuckCount)</pre>
</li>
</ul>
<a name="refreshConfig--">
@@ -434,7 +434,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockList">
<li class="blockList">
<h4>refreshConfig</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2108">refreshConfig</a>()</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2115">refreshConfig</a>()</pre>
</li>
</ul>
<a name="getTimeoutInterval--">
@@ -443,7 +443,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.h
<ul class="blockListLast">
<li class="blockList">
<h4>getTimeoutInterval</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2118">getTimeoutInterval</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerMonitor.html#line.2125">getTimeoutInterval</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.html#getTimeoutInterval--">getTimeoutInterval</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/InlineChore.html" title="class in org.apache.hadoop.hbase.procedure2">InlineChore</a></code></dd>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
index 38605e3..7c88bfd 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html
@@ -131,7 +131,7 @@ var activeTableTab = "activeTableTab";
</dl>
<hr>
<br>
-<pre>private class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1945">ProcedureExecutor.WorkerThread</a>
+<pre>private class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1952">ProcedureExecutor.WorkerThread</a>
extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThread.html" title="class in org.apache.hadoop.hbase.procedure2">StoppableThread</a></pre>
</li>
</ul>
@@ -282,7 +282,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>executionStartTime</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1946">executionStartTime</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1953">executionStartTime</a></pre>
</li>
</ul>
<a name="activeProcedure">
@@ -291,7 +291,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockListLast">
<li class="blockList">
<h4>activeProcedure</h4>
-<pre>private volatile <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1947">activeProcedure</a></pre>
+<pre>private volatile <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1954">activeProcedure</a></pre>
</li>
</ul>
</li>
@@ -308,7 +308,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>WorkerThread</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1949">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1956">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group)</pre>
</li>
</ul>
<a name="WorkerThread-java.lang.ThreadGroup-java.lang.String-">
@@ -317,7 +317,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockListLast">
<li class="blockList">
<h4>WorkerThread</h4>
-<pre>protected <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1953">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group,
+<pre>protected <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1960">WorkerThread</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> group,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> prefix)</pre>
</li>
</ul>
@@ -335,7 +335,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>sendStopSignal</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1959">sendStopSignal</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1966">sendStopSignal</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThread.html#sendStopSignal--">sendStopSignal</a></code> in class <code><a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThread.html" title="class in org.apache.hadoop.hbase.procedure2">StoppableThread</a></code></dd>
@@ -348,7 +348,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>run</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1963">run</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.1970">run</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true#run--" title="class or interface in java.lang">run</a></code> in interface <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Runnable.html?is-external=true" title="class or interface in java.lang">Runnable</a></code></dd>
@@ -363,7 +363,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>toString</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.2004">toString</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.2011">toString</a>()</pre>
<dl>
<dt><span class="overrideSpecifyLabel">Overrides:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true#toString--" title="class or interface in java.lang">toString</a></code> in class <code><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Thread.html?is-external=true" title="class or interface in java.lang">Thread</a></code></dd>
@@ -376,7 +376,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockList">
<li class="blockList">
<h4>getCurrentRunTime</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.2012">getCurrentRunTime</a>()</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.2019">getCurrentRunTime</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the time since the current procedure is running</dd>
@@ -389,7 +389,7 @@ extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/StoppableThre
<ul class="blockListLast">
<li class="blockList">
<h4>keepAlive</h4>
-<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.2017">keepAlive</a>(long lastUpdate)</pre>
+<pre>protected boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html#line.2024">keepAlive</a>(long lastUpdate)</pre>
</li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
index 430c656..5014748 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html
@@ -110,7 +110,7 @@ var activeTableTab = "activeTableTab";
<hr>
<br>
<pre>@InterfaceAudience.Private
-public class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.78">ProcedureExecutor</a><TEnvironment>
+public class <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.79">ProcedureExecutor</a><TEnvironment>
extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html?is-external=true" title="class or interface in java.lang">Object</a></pre>
<div class="block">Thread Pool that executes the submitted procedures.
The executor has a ProcedureStore associated.
@@ -777,7 +777,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>LOG</h4>
-<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.79">LOG</a></pre>
+<pre>private static final org.slf4j.Logger <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.80">LOG</a></pre>
</li>
</ul>
<a name="CHECK_OWNER_SET_CONF_KEY">
@@ -786,7 +786,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>CHECK_OWNER_SET_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.81">CHECK_OWNER_SET_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.82">CHECK_OWNER_SET_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.CHECK_OWNER_SET_CONF_KEY">Constant Field Values</a></dd>
@@ -799,7 +799,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_CHECK_OWNER_SET</h4>
-<pre>private static final boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.82">DEFAULT_CHECK_OWNER_SET</a></pre>
+<pre>private static final boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.83">DEFAULT_CHECK_OWNER_SET</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.DEFAULT_CHECK_OWNER_SET">Constant Field Values</a></dd>
@@ -812,7 +812,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>WORKER_KEEP_ALIVE_TIME_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.84">WORKER_KEEP_ALIVE_TIME_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.85">WORKER_KEEP_ALIVE_TIME_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.WORKER_KEEP_ALIVE_TIME_CONF_KEY">Constant Field Values</a></dd>
@@ -825,7 +825,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_WORKER_KEEP_ALIVE_TIME</h4>
-<pre>private static final long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.86">DEFAULT_WORKER_KEEP_ALIVE_TIME</a></pre>
+<pre>private static final long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.87">DEFAULT_WORKER_KEEP_ALIVE_TIME</a></pre>
</li>
</ul>
<a name="EVICT_TTL_CONF_KEY">
@@ -834,7 +834,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>EVICT_TTL_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.88">EVICT_TTL_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.89">EVICT_TTL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.EVICT_TTL_CONF_KEY">Constant Field Values</a></dd>
@@ -847,7 +847,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_EVICT_TTL</h4>
-<pre>static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.89">DEFAULT_EVICT_TTL</a></pre>
+<pre>static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.90">DEFAULT_EVICT_TTL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.DEFAULT_EVICT_TTL">Constant Field Values</a></dd>
@@ -860,7 +860,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>EVICT_ACKED_TTL_CONF_KEY</h4>
-<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.91">EVICT_ACKED_TTL_CONF_KEY</a></pre>
+<pre>public static final <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.92">EVICT_ACKED_TTL_CONF_KEY</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.EVICT_ACKED_TTL_CONF_KEY">Constant Field Values</a></dd>
@@ -873,7 +873,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>DEFAULT_ACKED_EVICT_TTL</h4>
-<pre>static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.92">DEFAULT_ACKED_EVICT_TTL</a></pre>
+<pre>static final int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.93">DEFAULT_ACKED_EVICT_TTL</a></pre>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.hadoop.hbase.procedure2.ProcedureExecutor.DEFAULT_ACKED_EVICT_TTL">Constant Field Values</a></dd>
@@ -886,7 +886,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>testing</h4>
-<pre>volatile <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.99">testing</a></pre>
+<pre>volatile <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.Testing</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.100">testing</a></pre>
<div class="block"><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#testing"><code>testing</code></a> is non-null when ProcedureExecutor is being tested. Tests will try to
break PE having it fail at various junctures. When non-null, testing is set to an instance of
the below internal <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html" title="class in org.apache.hadoop.hbase.procedure2"><code>ProcedureExecutor.Testing</code></a> class with flags set for the particular test.</div>
@@ -898,7 +898,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>completed</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure [...]
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/CompletedProcedureRetainer.html" title="class in org.apache.hadoop.hbase.procedure [...]
<div class="block">Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.
Once a Root-Procedure completes (success or failure), the result will be added to this map.
The user of ProcedureExecutor should call getResult(procId) to get the result.</div>
@@ -910,7 +910,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>rollbackStack</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootP [...]
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootP [...]
<div class="block">Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.
The RootProcedureState contains the execution stack of the Root-Procedure,
It is added to the map by submitProcedure() and removed on procedure completion.</div>
@@ -922,7 +922,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>procedures</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>& [...]
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>,<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a>& [...]
<div class="block">Helper map to lookup the live procedures by ID.
This map contains every procedure. root-procedures and subprocedures.</div>
</li>
@@ -933,7 +933,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>nonceKeysToProcIdsMap</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href=". [...]
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/ConcurrentHashMap.html?is-external=true" title="class or interface in java.util.concurrent">ConcurrentHashMap</a><<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a>,<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href=". [...]
<div class="block">Helper map to lookup whether the procedure already issued from the same client. This map
contains every root procedure.</div>
</li>
@@ -944,7 +944,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>listeners</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase [...]
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase [...]
</li>
</ul>
<a name="conf">
@@ -953,7 +953,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>conf</h4>
-<pre>private org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.194">conf</a></pre>
+<pre>private org.apache.hadoop.conf.Configuration <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.195">conf</a></pre>
</li>
</ul>
<a name="threadGroup">
@@ -962,7 +962,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>threadGroup</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.202">threadGroup</a></pre>
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/ThreadGroup.html?is-external=true" title="class or interface in java.lang">ThreadGroup</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.203">threadGroup</a></pre>
<div class="block">Created in the <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#init-int-boolean-"><code>init(int, boolean)</code></a> method. Destroyed in <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#join--"><code>join()</code></a> (FIX! Doing
resource handling rather than observing in a #join is unexpected).
Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery
@@ -975,7 +975,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>workerThreads</h4>
-<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#l [...]
+<pre>private <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CopyOnWriteArrayList.html?is-external=true" title="class or interface in java.util.concurrent">CopyOnWriteArrayList</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.WorkerThread.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.WorkerThread</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#l [...]
<div class="block">Created in the <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#init-int-boolean-"><code>init(int, boolean)</code></a> method. Terminated in <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#join--"><code>join()</code></a> (FIX! Doing
resource handling rather than observing in a #join is unexpected).
Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery
@@ -988,7 +988,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>timeoutExecutor</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.html" title="class in org.apache.hadoop.hbase.procedure2">TimeoutExecutorThread</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.218">timeoutExecutor</a></pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.html" title="class in org.apache.hadoop.hbase.procedure2">TimeoutExecutorThread</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.219">timeoutExecutor</a></pre>
<div class="block">Created in the <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#init-int-boolean-"><code>init(int, boolean)</code></a> method. Terminated in <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#join--"><code>join()</code></a> (FIX! Doing
resource handling rather than observing in a #join is unexpected).
Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery
@@ -1001,7 +1001,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>workerMonitorExecutor</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.html" title="class in org.apache.hadoop.hbase.procedure2">TimeoutExecutorThread</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.227">workerMonitorExecutor</a></pre>
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.html" title="class in org.apache.hadoop.hbase.procedure2">TimeoutExecutorThread</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.228">workerMonitorExecutor</a></pre>
<div class="block">WorkerMonitor check for stuck workers and new worker thread when necessary, for example if
there is no worker to assign meta, it will new worker thread for it, so it is very important.
TimeoutExecutor execute many tasks like DeadServerMetricRegionChore RegionInTransitionChore
@@ -1015,7 +1015,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>corePoolSize</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.229">corePoolSize</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.230">corePoolSize</a></pre>
</li>
</ul>
<a name="maxPoolSize">
@@ -1024,7 +1024,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>maxPoolSize</h4>
-<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.230">maxPoolSize</a></pre>
+<pre>private int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.231">maxPoolSize</a></pre>
</li>
</ul>
<a name="keepAliveTime">
@@ -1033,7 +1033,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>keepAliveTime</h4>
-<pre>private volatile long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.232">keepAliveTime</a></pre>
+<pre>private volatile long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.233">keepAliveTime</a></pre>
</li>
</ul>
<a name="scheduler">
@@ -1042,7 +1042,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>scheduler</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.237">scheduler</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.238">scheduler</a></pre>
<div class="block">Scheduler/Queue that contains runnable procedures.</div>
</li>
</ul>
@@ -1052,7 +1052,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>forceUpdateExecutor</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/Executor.html?is-external=true" title="class or interface in java.util.concurrent">Executor</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.239">forceUpdateExecutor</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/Executor.html?is-external=true" title="class or interface in java.util.concurrent">Executor</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.240">forceUpdateExecutor</a></pre>
</li>
</ul>
<a name="lastProcId">
@@ -1061,7 +1061,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>lastProcId</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.242">lastProcId</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.243">lastProcId</a></pre>
</li>
</ul>
<a name="workerId">
@@ -1070,7 +1070,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>workerId</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.243">workerId</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicLong.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicLong</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.244">workerId</a></pre>
</li>
</ul>
<a name="activeExecutorCount">
@@ -1079,7 +1079,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>activeExecutorCount</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.244">activeExecutorCount</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicInteger.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicInteger</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.245">activeExecutorCount</a></pre>
</li>
</ul>
<a name="running">
@@ -1088,7 +1088,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>running</h4>
-<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicBoolean.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicBoolean</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.245">running</a></pre>
+<pre>private final <a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/atomic/AtomicBoolean.html?is-external=true" title="class or interface in java.util.concurrent.atomic">AtomicBoolean</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.246">running</a></pre>
</li>
</ul>
<a name="environment">
@@ -1097,7 +1097,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>environment</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.246">environment</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.247">environment</a></pre>
</li>
</ul>
<a name="store">
@@ -1106,7 +1106,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>store</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.247">store</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.248">store</a></pre>
</li>
</ul>
<a name="checkOwnerSet">
@@ -1115,7 +1115,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>checkOwnerSet</h4>
-<pre>private final boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.249">checkOwnerSet</a></pre>
+<pre>private final boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.250">checkOwnerSet</a></pre>
</li>
</ul>
<a name="procExecutionLock">
@@ -1124,7 +1124,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>procExecutionLock</h4>
-<pre>private final <a href="../../../../../org/apache/hadoop/hbase/util/IdLock.html" title="class in org.apache.hadoop.hbase.util">IdLock</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.257">procExecutionLock</a></pre>
+<pre>private final <a href="../../../../../org/apache/hadoop/hbase/util/IdLock.html" title="class in org.apache.hadoop.hbase.util">IdLock</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.258">procExecutionLock</a></pre>
</li>
</ul>
</li>
@@ -1143,7 +1143,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>ProcedureExecutor</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.259">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.260">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> environment,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> store)</pre>
</li>
@@ -1156,7 +1156,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>ProcedureExecutor</h4>
-<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.301">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
+<pre>public <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.302">ProcedureExecutor</a>(org.apache.hadoop.conf.Configuration conf,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> environment,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> store,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> scheduler)</pre>
@@ -1176,7 +1176,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isRootFinished</h4>
-<pre>private boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.264">isRootFinished</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><?> proc)</pre>
+<pre>private boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.265">isRootFinished</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><?> proc)</pre>
</li>
</ul>
<a name="forceUpdateProcedure-long-">
@@ -1185,7 +1185,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>forceUpdateProcedure</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.269">forceUpdateProcedure</a>(long procId)
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.270">forceUpdateProcedure</a>(long procId)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1199,7 +1199,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>load</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.324">load</a>(boolean abortOnCorruption)
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.325">load</a>(boolean abortOnCorruption)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
<dt><span class="throwsLabel">Throws:</span></dt>
@@ -1213,7 +1213,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>restoreLock</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.357">restoreLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.358">restoreLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> restored)</pre>
</li>
</ul>
@@ -1223,7 +1223,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>restoreLocks</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.362">restoreLocks</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ [...]
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.363">restoreLocks</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Deque.html?is-external=true" title="class or interface in java.util">Deque</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ [...]
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> restored)</pre>
</li>
</ul>
@@ -1233,7 +1233,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>restoreLocks</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.381">restoreLocks</a>()</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.382">restoreLocks</a>()</pre>
</li>
</ul>
<a name="loadProcedures-org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator-boolean-">
@@ -1242,7 +1242,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>loadProcedures</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.401">loadProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a> procIter,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.402">loadProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.ProcedureIterator.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore.ProcedureIterator</a> procIter,
boolean abortOnCorruption)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<dl>
@@ -1257,7 +1257,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>init</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.563">init</a>(int numThreads,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.564">init</a>(int numThreads,
boolean abortOnCorruption)
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Initialize the procedure executor, but do not start workers. We will start them later.
@@ -1281,7 +1281,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>startWorkers</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.609">startWorkers</a>()
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.610">startWorkers</a>()
throws <a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Start the workers.</div>
<dl>
@@ -1296,7 +1296,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>stop</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.630">stop</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.631">stop</a>()</pre>
</li>
</ul>
<a name="join--">
@@ -1305,7 +1305,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>join</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.642">join</a>()</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.643">join</a>()</pre>
</li>
</ul>
<a name="refreshConfiguration-org.apache.hadoop.conf.Configuration-">
@@ -1314,7 +1314,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>refreshConfiguration</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.675">refreshConfiguration</a>(org.apache.hadoop.conf.Configuration conf)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.676">refreshConfiguration</a>(org.apache.hadoop.conf.Configuration conf)</pre>
</li>
</ul>
<a name="isRunning--">
@@ -1323,7 +1323,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isRunning</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.684">isRunning</a>()</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.685">isRunning</a>()</pre>
</li>
</ul>
<a name="getWorkerThreadCount--">
@@ -1332,7 +1332,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getWorkerThreadCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.691">getWorkerThreadCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.692">getWorkerThreadCount</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the current number of worker threads.</dd>
@@ -1345,7 +1345,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getCorePoolSize</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.698">getCorePoolSize</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.699">getCorePoolSize</a>()</pre>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the core pool size settings.</dd>
@@ -1358,7 +1358,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getActiveExecutorCount</h4>
-<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.702">getActiveExecutorCount</a>()</pre>
+<pre>public int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.703">getActiveExecutorCount</a>()</pre>
</li>
</ul>
<a name="getEnvironment--">
@@ -1367,7 +1367,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getEnvironment</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.706">getEnvironment</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.707">getEnvironment</a>()</pre>
</li>
</ul>
<a name="getStore--">
@@ -1376,7 +1376,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getStore</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.710">getStore</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/store/ProcedureStore.html" title="interface in org.apache.hadoop.hbase.procedure2.store">ProcedureStore</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.711">getStore</a>()</pre>
</li>
</ul>
<a name="getScheduler--">
@@ -1385,7 +1385,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getScheduler</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.714">getScheduler</a>()</pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.715">getScheduler</a>()</pre>
</li>
</ul>
<a name="setKeepAliveTime-long-java.util.concurrent.TimeUnit-">
@@ -1394,7 +1394,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setKeepAliveTime</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.718">setKeepAliveTime</a>(long keepAliveTime,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.719">setKeepAliveTime</a>(long keepAliveTime,
<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</pre>
</li>
</ul>
@@ -1404,7 +1404,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getKeepAliveTime</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.723">getKeepAliveTime</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.724">getKeepAliveTime</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/TimeUnit.html?is-external=true" title="class or interface in java.util.concurrent">TimeUnit</a> timeUnit)</pre>
</li>
</ul>
<a name="addChore-org.apache.hadoop.hbase.procedure2.ProcedureInMemoryChore-">
@@ -1413,7 +1413,8 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>addChore</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.735">addChore</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.736">addChore</a>(@Nullable
+ <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
<div class="block">Add a chore procedure to the executor</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1427,7 +1428,8 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>removeChore</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.745">removeChore</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.749">removeChore</a>(@Nullable
+ <a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureInMemoryChore.html" title="class in org.apache.hadoop.hbase.procedure2">ProcedureInMemoryChore</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> chore)</pre>
<div class="block">Remove a chore procedure from the executor</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1443,7 +1445,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>createNonceKey</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.759">createNonceKey</a>(long nonceGroup,
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.766">createNonceKey</a>(long nonceGroup,
long nonce)</pre>
<div class="block">Create a NonceKey from the specified nonceGroup and nonce.</div>
<dl>
@@ -1461,7 +1463,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>registerNonce</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.774">registerNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.781">registerNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
<div class="block">Register a nonce for a procedure that is going to be submitted.
A procId will be reserved and on submitProcedure(),
the procedure with the specified nonce will take the reserved ProcId.
@@ -1482,7 +1484,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>unregisterNonceIfProcedureWasNotSubmitted</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.809">unregisterNonceIfProcedureWasNotSubmitted</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.816">unregisterNonceIfProcedureWasNotSubmitted</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
<div class="block">Remove the NonceKey if the procedure was not submitted to the executor.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1496,7 +1498,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>setFailureResultForNonce</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.834">setFailureResultForNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey,
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.841">setFailureResultForNonce</a>(<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> procName,
<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a> procOwner,
<a href="https://docs.oracle.com/javase/8/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a> exception)</pre>
@@ -1517,7 +1519,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedure</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.861">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.868">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
<div class="block">Add a new root-procedure to the executor.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1533,7 +1535,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>bypassProcedure</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.900">bypassProcedure</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util [...]
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Boolean.html?is-external=true" title="class or interface in java.lang">Boolean</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.907">bypassProcedure</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/util [...]
long lockWait,
boolean force,
boolean recursive)
@@ -1584,7 +1586,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>bypassProcedure</h4>
-<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.910">bypassProcedure</a>(long pid,
+<pre>boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.917">bypassProcedure</a>(long pid,
long lockWait,
boolean override,
boolean recursive)
@@ -1601,7 +1603,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedure</h4>
-<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1025">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
+<pre>public long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1032">submitProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="../../../../../org/apache/hadoop/hbase/util/NonceKey.html" title="class in org.apache.hadoop.hbase.util">NonceKey</a> nonceKey)</pre>
<div class="block">Add a new root-procedure to the executor.</div>
<dl>
@@ -1619,7 +1621,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitProcedures</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1056">submitProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] procs)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1063">submitProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] procs)</pre>
<div class="block">Add a set of new root-procedure to the executor.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1633,7 +1635,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>prepareProcedure</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1079">prepareProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/proced [...]
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1086">prepareProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/proced [...]
</li>
</ul>
<a name="pushProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1642,7 +1644,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>pushProcedure</h4>
-<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1088">pushProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1095">pushProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="abort-long-">
@@ -1651,7 +1653,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>abort</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1112">abort</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1119">abort</a>(long procId)</pre>
<div class="block">Send an abort notification the specified procedure.
Depending on the procedure implementation the abort can be considered or ignored.</div>
<dl>
@@ -1668,7 +1670,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>abort</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1123">abort</a>(long procId,
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1130">abort</a>(long procId,
boolean mayInterruptIfRunning)</pre>
<div class="block">Send an abort notification to the specified procedure.
Depending on the procedure implementation, the abort can be considered or ignored.</div>
@@ -1687,7 +1689,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedure</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1137">getProcedure</a>(long procId)</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1144">getProcedure</a>(long procId)</pre>
</li>
</ul>
<a name="getProcedure-java.lang.Class-long-">
@@ -1696,7 +1698,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedure</h4>
-<pre>public <T extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> T <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1141">getProcedure</a>(<a href="https://docs.oracle.com/j [...]
+<pre>public <T extends <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> T <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1148">getProcedure</a>(<a href="https://docs.oracle.com/j [...]
long procId)</pre>
</li>
</ul>
@@ -1706,7 +1708,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getResult</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1149">getResult</a>(long procId)</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1156">getResult</a>(long procId)</pre>
</li>
</ul>
<a name="isFinished-long-">
@@ -1715,7 +1717,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isFinished</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1165">isFinished</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1172">isFinished</a>(long procId)</pre>
<div class="block">Return true if the procedure is finished.
The state may be "completed successfully" or "failed and rolledback".
Use getResult() to check the state or get the result data.</div>
@@ -1733,7 +1735,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isStarted</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1174">isStarted</a>(long procId)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1181">isStarted</a>(long procId)</pre>
<div class="block">Return true if the procedure is started.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1749,7 +1751,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>removeResult</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1186">removeResult</a>(long procId)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1193">removeResult</a>(long procId)</pre>
<div class="block">Mark the specified completed procedure, as ready to remove.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
@@ -1763,7 +1765,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getResultOrProcedure</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1198">getResultOrProcedure</a>(long procId)</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1205">getResultOrProcedure</a>(long procId)</pre>
</li>
</ul>
<a name="isProcedureOwner-long-org.apache.hadoop.hbase.security.User-">
@@ -1772,7 +1774,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>isProcedureOwner</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1214">isProcedureOwner</a>(long procId,
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1221">isProcedureOwner</a>(long procId,
<a href="../../../../../org/apache/hadoop/hbase/security/User.html" title="class in org.apache.hadoop.hbase.security">User</a> user)</pre>
<div class="block">Check if the user is this procedure's owner</div>
<dl>
@@ -1791,7 +1793,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getActiveProceduresNoCopy</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../ [...]
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Collection.html?is-external=true" title="class or interface in java.util">Collection</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../ [...]
<div class="block">Should only be used when starting up, where the procedure workers have not been started.
<p/>
If the procedure works has been started, the return values maybe changed when you are
@@ -1805,7 +1807,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedures</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/ [...]
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>> <a href="../../../../../src-html/ [...]
<div class="block">Get procedures.</div>
<dl>
<dt><span class="returnLabel">Returns:</span></dt>
@@ -1819,7 +1821,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>registerListener</h4>
-<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1264">registerListener</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> listener)</pre>
+<pre>public void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1271">registerListener</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> listener)</pre>
</li>
</ul>
<a name="unregisterListener-org.apache.hadoop.hbase.procedure2.ProcedureExecutor.ProcedureExecutorListener-">
@@ -1828,7 +1830,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>unregisterListener</h4>
-<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1268">unregisterListener</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> listener)</pre>
+<pre>public boolean <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1275">unregisterListener</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureExecutor.ProcedureExecutorListener</a> listener)</pre>
</li>
</ul>
<a name="sendProcedureLoadedNotification-long-">
@@ -1837,7 +1839,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>sendProcedureLoadedNotification</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1272">sendProcedureLoadedNotification</a>(long procId)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1279">sendProcedureLoadedNotification</a>(long procId)</pre>
</li>
</ul>
<a name="sendProcedureAddedNotification-long-">
@@ -1846,7 +1848,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>sendProcedureAddedNotification</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1284">sendProcedureAddedNotification</a>(long procId)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1291">sendProcedureAddedNotification</a>(long procId)</pre>
</li>
</ul>
<a name="sendProcedureFinishedNotification-long-">
@@ -1855,7 +1857,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>sendProcedureFinishedNotification</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1296">sendProcedureFinishedNotification</a>(long procId)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1303">sendProcedureFinishedNotification</a>(long procId)</pre>
</li>
</ul>
<a name="nextProcId--">
@@ -1864,7 +1866,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>nextProcId</h4>
-<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1311">nextProcId</a>()</pre>
+<pre>private long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1318">nextProcId</a>()</pre>
</li>
</ul>
<a name="getLastProcId--">
@@ -1873,7 +1875,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getLastProcId</h4>
-<pre>protected long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1329">getLastProcId</a>()</pre>
+<pre>protected long <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1336">getLastProcId</a>()</pre>
</li>
</ul>
<a name="getActiveProcIds--">
@@ -1882,7 +1884,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getActiveProcIds</h4>
-<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1334">getActiveProcIds</a>()</pre>
+<pre>public <a href="https://docs.oracle.com/javase/8/docs/api/java/util/Set.html?is-external=true" title="class or interface in java.util">Set</a><<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1341">getActiveProcIds</a>()</pre>
</li>
</ul>
<a name="getRootProcedureId-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1891,7 +1893,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getRootProcedureId</h4>
-<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1338">getRootProcedureId</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecuto [...]
+<pre><a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Long.html?is-external=true" title="class or interface in java.lang">Long</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1345">getRootProcedureId</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecuto [...]
</li>
</ul>
<a name="executeProcedure-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1900,7 +1902,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>executeProcedure</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1345">executeProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1352">executeProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="acquireLock-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1909,7 +1911,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>acquireLock</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2">Procedure.LockState</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1441">acquireLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase [...]
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2">Procedure.LockState</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1448">acquireLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase [...]
</li>
</ul>
<a name="releaseLock-org.apache.hadoop.hbase.procedure2.Procedure-boolean-">
@@ -1918,7 +1920,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>releaseLock</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1451">releaseLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1458">releaseLock</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
boolean force)</pre>
</li>
</ul>
@@ -1928,7 +1930,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>executeRollback</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2">Procedure.LockState</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1464">executeRollback</a>(long rootProcId,
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2">Procedure.LockState</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1471">executeRollback</a>(long rootProcId,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack)</pre>
<div class="block">Execute the rollback of the full procedure stack. Once the procedure is rolledback, the
root-procedure will be visible as finished to user, and the result will be the fatal exception.</div>
@@ -1940,7 +1942,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>cleanupAfterRollbackOneStep</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1553">cleanupAfterRollbackOneStep</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1560">cleanupAfterRollbackOneStep</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="executeRollback-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1949,7 +1951,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>executeRollback</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2">Procedure.LockState</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1583">executeRollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/h [...]
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2">Procedure.LockState</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1590">executeRollback</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/h [...]
<div class="block">Execute the rollback of the procedure step.
It updates the store with the new state (stack index)
or will remove completly the procedure in case it is a child.</div>
@@ -1961,7 +1963,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>yieldProcedure</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1611">yieldProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1618">yieldProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="execProcedure-org.apache.hadoop.hbase.procedure2.RootProcedureState-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -1970,7 +1972,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>execProcedure</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1645">execProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1652">execProcedure</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procedure)</pre>
<div class="block">Executes <code>procedure</code>
<ul>
@@ -2007,7 +2009,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>kill</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1773">kill</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> msg)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1780">kill</a>(<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</a> msg)</pre>
</li>
</ul>
<a name="initializeChildren-org.apache.hadoop.hbase.procedure2.RootProcedureState-org.apache.hadoop.hbase.procedure2.Procedure-org.apache.hadoop.hbase.procedure2.Procedure:A-">
@@ -2016,7 +2018,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>initializeChildren</h4>
-<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1779">initializeChildren</a>(<a href="../../../../../org/apache/hadoop/hbase/pr [...]
+<pre>private <a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1786">initializeChildren</a>(<a href="../../../../../org/apache/hadoop/hbase/pr [...]
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procedure,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] subprocs)</pre>
</li>
@@ -2027,7 +2029,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>submitChildrenProcedures</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1815">submitChildrenProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] subprocs)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1822">submitChildrenProcedures</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] subprocs)</pre>
</li>
</ul>
<a name="countDownChildren-org.apache.hadoop.hbase.procedure2.RootProcedureState-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -2036,7 +2038,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>countDownChildren</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1825">countDownChildren</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1832">countDownChildren</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procedure)</pre>
</li>
</ul>
@@ -2046,7 +2048,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>updateStoreOnExec</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1844">updateStoreOnExec</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1851">updateStoreOnExec</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procStack,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> procedure,
<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>>[] subprocs)</pre>
</li>
@@ -2057,7 +2059,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>handleInterruptedException</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1870">handleInterruptedException</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1877">handleInterruptedException</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc,
<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/InterruptedException.html?is-external=true" title="class or interface in java.lang">InterruptedException</a> e)</pre>
</li>
</ul>
@@ -2067,7 +2069,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>execCompletionCleanup</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1879">execCompletionCleanup</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1886">execCompletionCleanup</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="procedureFinished-org.apache.hadoop.hbase.procedure2.Procedure-">
@@ -2076,7 +2078,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>procedureFinished</h4>
-<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1896">procedureFinished</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
+<pre>private void <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1903">procedureFinished</a>(<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.html" title="class in org.apache.hadoop.hbase.procedure2">Procedure</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> proc)</pre>
</li>
</ul>
<a name="getProcStack-long-">
@@ -2085,7 +2087,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcStack</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1923">getProcStack</a>(long rootProcId)</pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.html" title="class in org.apache.hadoop.hbase.procedure2">RootProcedureState</a><<a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html" title="type parameter in ProcedureExecutor">TEnvironment</a>> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1930">getProcStack</a>(long rootProcId)</pre>
</li>
</ul>
<a name="getProcedureScheduler--">
@@ -2094,7 +2096,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getProcedureScheduler</h4>
-<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1928">getProcedureScheduler</a>()</pre>
+<pre><a href="../../../../../org/apache/hadoop/hbase/procedure2/ProcedureScheduler.html" title="interface in org.apache.hadoop.hbase.procedure2">ProcedureScheduler</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1935">getProcedureScheduler</a>()</pre>
</li>
</ul>
<a name="getCompletedSize--">
@@ -2103,7 +2105,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockList">
<li class="blockList">
<h4>getCompletedSize</h4>
-<pre>int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1933">getCompletedSize</a>()</pre>
+<pre>int <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1940">getCompletedSize</a>()</pre>
</li>
</ul>
<a name="getProcExecutionLock--">
@@ -2112,7 +2114,7 @@ extends <a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html
<ul class="blockListLast">
<li class="blockList">
<h4>getProcExecutionLock</h4>
-<pre>public <a href="../../../../../org/apache/hadoop/hbase/util/IdLock.html" title="class in org.apache.hadoop.hbase.util">IdLock</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1938">getProcExecutionLock</a>()</pre>
+<pre>public <a href="../../../../../org/apache/hadoop/hbase/util/IdLock.html" title="class in org.apache.hadoop.hbase.util">IdLock</a> <a href="../../../../../src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.html#line.1945">getProcExecutionLock</a>()</pre>
</li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html b/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
index 2b4cad4..15be7b5 100644
--- a/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/procedure2/package-tree.html
@@ -216,11 +216,11 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">Procedure.LockState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockedResourceType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockedResourceType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/StateMachineProcedure.Flow.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">StateMachineProcedure.Flow</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/Procedure.LockState.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">Procedure.LockState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/RootProcedureState.State.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">RootProcedureState.State</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.procedure2.<a href="../../../../../org/apache/hadoop/hbase/procedure2/LockType.html" title="enum in org.apache.hadoop.hbase.procedure2"><span class="typeNameLink">LockType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html b/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
index 2df5311..d272154 100644
--- a/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/quotas/package-tree.html
@@ -240,12 +240,12 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">SpaceViolationPolicy</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaScope.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaScope</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/ThrottleType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">ThrottleType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/OperationQuota.OperationType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">OperationQuota.OperationType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaScope.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaScope</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/SpaceViolationPolicy.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">SpaceViolationPolicy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/RpcThrottlingException.Type.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">RpcThrottlingException.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.quotas.<a href="../../../../../org/apache/hadoop/hbase/quotas/QuotaType.html" title="enum in org.apache.hadoop.hbase.quotas"><span class="typeNameLink">QuotaType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
index f2f2216..96b43c9 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/package-tree.html
@@ -735,20 +735,20 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">CompactingMemStore.IndexType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.LimitScope.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.LimitScope</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChunkCreator.ChunkType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ChunkCreator.ChunkType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScanType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/Region.Operation.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">Region.Operation</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.StepDirection.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">DefaultHeapMemoryTuner.StepDirection</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.NextState.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.NextState</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/SplitLogWorker.TaskExecutor.Status.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">SplitLogWorker.TaskExecutor.Status</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.Type.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TimeRangeTracker.Type</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.Action.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MemStoreCompactionStrategy.Action</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/FlushType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">FlushType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/BloomType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">BloomType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScannerContext.LimitScope.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScannerContext.LimitScope</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ScanType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ScanType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/HRegion.FlushResult.Result.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">HRegion.FlushResult.Result</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MetricsRegionServerSourceFactoryImpl.FactoryStorage</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/BloomType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">BloomType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.Action.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">MemStoreCompactionStrategy.Action</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/Region.Operation.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">Region.Operation</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.StepDirection.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">DefaultHeapMemoryTuner.StepDirection</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/CompactingMemStore.IndexType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">CompactingMemStore.IndexType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/TimeRangeTracker.Type.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">TimeRangeTracker.Type</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.<a href="../../../../../org/apache/hadoop/hbase/regionserver/ChunkCreator.ChunkType.html" title="enum in org.apache.hadoop.hbase.regionserver"><span class="typeNameLink">ChunkCreator.ChunkType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
index ba24945..e1c6d6d 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/querymatcher/package-tree.html
@@ -131,8 +131,8 @@
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/ScanQueryMatcher.MatchCode.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">ScanQueryMatcher.MatchCode</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/DeleteTracker.DeleteResult.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">DeleteTracker.DeleteResult</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/StripeCompactionScanQueryMatcher.DropDeletesInOutput.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">StripeCompactionScanQueryMatcher.DropDeletesInOutput</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.querymatcher.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/querymatcher/DeleteTracker.DeleteResult.html" title="enum in org.apache.hadoop.hbase.regionserver.querymatcher"><span class="typeNameLink">DeleteTracker.DeleteResult</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html b/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
index 84a970c..b1582b5 100644
--- a/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/regionserver/wal/package-tree.html
@@ -248,9 +248,9 @@
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/CompressionContext.DictionaryIndex.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">CompressionContext.DictionaryIndex</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/RingBufferTruck.Type.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">RingBufferTruck.Type</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/WALActionsListener.RollRequestReason.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">WALActionsListener.RollRequestReason</span></a></li>
<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">ProtobufLogReader.WALHdrResult</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.regionserver.wal.<a href="../../../../../../org/apache/hadoop/hbase/regionserver/wal/RingBufferTruck.Type.html" title="enum in org.apache.hadoop.hbase.regionserver.wal"><span class="typeNameLink">RingBufferTruck.Type</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html b/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
index c13ca76..d1bee32 100644
--- a/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/security/access/package-tree.html
@@ -162,12 +162,12 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
+<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/Permission.Scope.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">Permission.Scope</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/Permission.Action.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">Permission.Action</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclHelper.HDFSAclOperation.OperationType.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">SnapshotScannerHDFSAclHelper.HDFSAclOperation.OperationType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclHelper.HDFSAclOperation.AclType.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">SnapshotScannerHDFSAclHelper.HDFSAclOperation.AclType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/AccessControlFilter.Strategy.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">AccessControlFilter.Strategy</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/AccessController.OpType.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">AccessController.OpType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclHelper.HDFSAclOperation.AclType.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">SnapshotScannerHDFSAclHelper.HDFSAclOperation.AclType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/Permission.Scope.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">Permission.Scope</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.access.<a href="../../../../../../org/apache/hadoop/hbase/security/access/SnapshotScannerHDFSAclHelper.HDFSAclOperation.OperationType.html" title="enum in org.apache.hadoop.hbase.security.access"><span class="typeNameLink">SnapshotScannerHDFSAclHelper.HDFSAclOperation.OperationType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/security/package-tree.html b/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
index ffdb431..c060011 100644
--- a/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/security/package-tree.html
@@ -189,8 +189,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/AuthMethod.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">AuthMethod</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslStatus.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslStatus</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/AuthMethod.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">AuthMethod</span></a></li>
<li type="circle">org.apache.hadoop.hbase.security.<a href="../../../../../org/apache/hadoop/hbase/security/SaslUtil.QualityOfProtection.html" title="enum in org.apache.hadoop.hbase.security"><span class="typeNameLink">SaslUtil.QualityOfProtection</span></a></li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html b/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
index e069dc4..6d46ba3 100644
--- a/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/thrift/package-tree.html
@@ -211,8 +211,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">MetricsThriftServerSourceFactoryImpl.FactoryStorage</span></a></li>
<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ThriftMetrics.ThriftServerType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ThriftMetrics.ThriftServerType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/MetricsThriftServerSourceFactoryImpl.FactoryStorage.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">MetricsThriftServerSourceFactoryImpl.FactoryStorage</span></a></li>
<li type="circle">org.apache.hadoop.hbase.thrift.<a href="../../../../../org/apache/hadoop/hbase/thrift/ImplType.html" title="enum in org.apache.hadoop.hbase.thrift"><span class="typeNameLink">ImplType</span></a></li>
</ul>
</li>
diff --git a/devapidocs/org/apache/hadoop/hbase/util/package-tree.html b/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
index 43c233f..e1a511b 100644
--- a/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/util/package-tree.html
@@ -559,14 +559,14 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Order.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Order</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/HbckErrorReporter.ERROR_CODE.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">HbckErrorReporter.ERROR_CODE</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PrettyPrinter.Unit.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PrettyPrinter.Unit</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.PureJavaComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.PureJavaComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a><T>)</li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/IdReadWriteLockWithObjectPool.ReferenceType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">IdReadWriteLockWithObjectPool.ReferenceType</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Order.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Order</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PoolMap.PoolType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PoolMap.PoolType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.LexicographicalComparerHolder.UnsafeComparer.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">Bytes.LexicographicalComparerHolder.UnsafeComparer</span></a> (implements org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/Bytes.Comparer.html" title="interface in org.apache.hadoop.hbase.util">Bytes.Comparer</a><T>)</li>
+<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/IdReadWriteLockWithObjectPool.ReferenceType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">IdReadWriteLockWithObjectPool.ReferenceType</span></a></li>
<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/ChecksumType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">ChecksumType</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/HbckErrorReporter.ERROR_CODE.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">HbckErrorReporter.ERROR_CODE</span></a></li>
-<li type="circle">org.apache.hadoop.hbase.util.<a href="../../../../../org/apache/hadoop/hbase/util/PoolMap.PoolType.html" title="enum in org.apache.hadoop.hbase.util"><span class="typeNameLink">PoolMap.PoolType</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html b/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
index f2c94bd..d78cc8b 100644
--- a/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
+++ b/devapidocs/org/apache/hadoop/hbase/wal/package-tree.html
@@ -199,8 +199,8 @@
<ul>
<li type="circle">java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Enum.html?is-external=true" title="class or interface in java.lang"><span class="typeNameLink">Enum</span></a><E> (implements java.lang.<a href="https://docs.oracle.com/javase/8/docs/api/java/lang/Comparable.html?is-external=true" title="class or interface in java.lang">Comparable</a><T>, java.io.<a href="https://docs.oracle.com/javase/8/docs/api/java/io/Serializable.html?is-external=true [...]
<ul>
-<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/RegionGroupingProvider.Strategies.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">RegionGroupingProvider.Strategies</span></a></li>
<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/WALFactory.Providers.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">WALFactory.Providers</span></a></li>
+<li type="circle">org.apache.hadoop.hbase.wal.<a href="../../../../../org/apache/hadoop/hbase/wal/RegionGroupingProvider.Strategies.html" title="enum in org.apache.hadoop.hbase.wal"><span class="typeNameLink">RegionGroupingProvider.Strategies</span></a></li>
</ul>
</li>
</ul>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
index 1a54976..cb651d0 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/Version.html
@@ -18,9 +18,9 @@
<span class="sourceLineNo">010</span> justification="Intentional; to be modified in test")<a name="line.10"></a>
<span class="sourceLineNo">011</span>public class Version {<a name="line.11"></a>
<span class="sourceLineNo">012</span> public static final String version = new String("3.0.0-SNAPSHOT");<a name="line.12"></a>
-<span class="sourceLineNo">013</span> public static final String revision = "546828685e2a7c2d686925a80c7a68dd4d4e3f8d";<a name="line.13"></a>
+<span class="sourceLineNo">013</span> public static final String revision = "24823ecfc9ddc9bbb473657d2ccf73cbd971ee2b";<a name="line.13"></a>
<span class="sourceLineNo">014</span> public static final String user = "jenkins";<a name="line.14"></a>
-<span class="sourceLineNo">015</span> public static final String date = "Sat Feb 8 14:38:19 UTC 2020";<a name="line.15"></a>
+<span class="sourceLineNo">015</span> public static final String date = "Sun Feb 9 14:38:57 UTC 2020";<a name="line.15"></a>
<span class="sourceLineNo">016</span> public static final String url = "git://jenkins-websites-he-de.apache.org/home/jenkins/jenkins-slave/workspace/hbase_generate_website/hbase";<a name="line.16"></a>
<span class="sourceLineNo">017</span> public static final String srcChecksum = "(stdin)=";<a name="line.17"></a>
<span class="sourceLineNo">018</span>}<a name="line.18"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
index 7816bab..95831a0 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.KeepAliveWorkerThread.html
@@ -25,2109 +25,2116 @@
<span class="sourceLineNo">017</span> */<a name="line.17"></a>
<span class="sourceLineNo">018</span>package org.apache.hadoop.hbase.procedure2;<a name="line.18"></a>
<span class="sourceLineNo">019</span><a name="line.19"></a>
-<span class="sourceLineNo">020</span>import java.io.IOException;<a name="line.20"></a>
-<span class="sourceLineNo">021</span>import java.io.UncheckedIOException;<a name="line.21"></a>
-<span class="sourceLineNo">022</span>import java.util.ArrayDeque;<a name="line.22"></a>
-<span class="sourceLineNo">023</span>import java.util.ArrayList;<a name="line.23"></a>
-<span class="sourceLineNo">024</span>import java.util.Arrays;<a name="line.24"></a>
-<span class="sourceLineNo">025</span>import java.util.Collection;<a name="line.25"></a>
-<span class="sourceLineNo">026</span>import java.util.Deque;<a name="line.26"></a>
-<span class="sourceLineNo">027</span>import java.util.HashSet;<a name="line.27"></a>
-<span class="sourceLineNo">028</span>import java.util.List;<a name="line.28"></a>
-<span class="sourceLineNo">029</span>import java.util.Set;<a name="line.29"></a>
-<span class="sourceLineNo">030</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.30"></a>
-<span class="sourceLineNo">031</span>import java.util.concurrent.CopyOnWriteArrayList;<a name="line.31"></a>
-<span class="sourceLineNo">032</span>import java.util.concurrent.Executor;<a name="line.32"></a>
-<span class="sourceLineNo">033</span>import java.util.concurrent.Executors;<a name="line.33"></a>
-<span class="sourceLineNo">034</span>import java.util.concurrent.TimeUnit;<a name="line.34"></a>
-<span class="sourceLineNo">035</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.35"></a>
-<span class="sourceLineNo">036</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.36"></a>
-<span class="sourceLineNo">037</span>import java.util.concurrent.atomic.AtomicLong;<a name="line.37"></a>
-<span class="sourceLineNo">038</span>import java.util.stream.Collectors;<a name="line.38"></a>
-<span class="sourceLineNo">039</span>import java.util.stream.Stream;<a name="line.39"></a>
-<span class="sourceLineNo">040</span>import org.apache.hadoop.conf.Configuration;<a name="line.40"></a>
-<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.HConstants;<a name="line.41"></a>
-<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;<a name="line.42"></a>
-<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.log.HBaseMarkers;<a name="line.43"></a>
-<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.procedure2.Procedure.LockState;<a name="line.44"></a>
-<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;<a name="line.45"></a>
-<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;<a name="line.46"></a>
-<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureStoreListener;<a name="line.47"></a>
-<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.procedure2.util.StringUtils;<a name="line.48"></a>
-<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.security.User;<a name="line.49"></a>
-<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.50"></a>
-<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.util.IdLock;<a name="line.51"></a>
-<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.util.NonceKey;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.53"></a>
-<span class="sourceLineNo">054</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.54"></a>
-<span class="sourceLineNo">055</span>import org.slf4j.Logger;<a name="line.55"></a>
-<span class="sourceLineNo">056</span>import org.slf4j.LoggerFactory;<a name="line.56"></a>
-<span class="sourceLineNo">057</span><a name="line.57"></a>
-<span class="sourceLineNo">058</span>import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;<a name="line.58"></a>
-<span class="sourceLineNo">059</span>import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;<a name="line.59"></a>
-<span class="sourceLineNo">060</span>import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;<a name="line.60"></a>
-<span class="sourceLineNo">061</span><a name="line.61"></a>
-<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.62"></a>
-<span class="sourceLineNo">063</span><a name="line.63"></a>
-<span class="sourceLineNo">064</span>/**<a name="line.64"></a>
-<span class="sourceLineNo">065</span> * Thread Pool that executes the submitted procedures.<a name="line.65"></a>
-<span class="sourceLineNo">066</span> * The executor has a ProcedureStore associated.<a name="line.66"></a>
-<span class="sourceLineNo">067</span> * Each operation is logged and on restart the pending procedures are resumed.<a name="line.67"></a>
-<span class="sourceLineNo">068</span> *<a name="line.68"></a>
-<span class="sourceLineNo">069</span> * Unless the Procedure code throws an error (e.g. invalid user input)<a name="line.69"></a>
-<span class="sourceLineNo">070</span> * the procedure will complete (at some point in time), On restart the pending<a name="line.70"></a>
-<span class="sourceLineNo">071</span> * procedures are resumed and the once failed will be rolledback.<a name="line.71"></a>
-<span class="sourceLineNo">072</span> *<a name="line.72"></a>
-<span class="sourceLineNo">073</span> * The user can add procedures to the executor via submitProcedure(proc)<a name="line.73"></a>
-<span class="sourceLineNo">074</span> * check for the finished state via isFinished(procId)<a name="line.74"></a>
-<span class="sourceLineNo">075</span> * and get the result via getResult(procId)<a name="line.75"></a>
-<span class="sourceLineNo">076</span> */<a name="line.76"></a>
-<span class="sourceLineNo">077</span>@InterfaceAudience.Private<a name="line.77"></a>
-<span class="sourceLineNo">078</span>public class ProcedureExecutor<TEnvironment> {<a name="line.78"></a>
-<span class="sourceLineNo">079</span> private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);<a name="line.79"></a>
-<span class="sourceLineNo">080</span><a name="line.80"></a>
-<span class="sourceLineNo">081</span> public static final String CHECK_OWNER_SET_CONF_KEY = "hbase.procedure.check.owner.set";<a name="line.81"></a>
-<span class="sourceLineNo">082</span> private static final boolean DEFAULT_CHECK_OWNER_SET = false;<a name="line.82"></a>
-<span class="sourceLineNo">083</span><a name="line.83"></a>
-<span class="sourceLineNo">084</span> public static final String WORKER_KEEP_ALIVE_TIME_CONF_KEY =<a name="line.84"></a>
-<span class="sourceLineNo">085</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.85"></a>
-<span class="sourceLineNo">086</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.86"></a>
-<span class="sourceLineNo">087</span><a name="line.87"></a>
-<span class="sourceLineNo">088</span> public static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.88"></a>
-<span class="sourceLineNo">089</span> static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.89"></a>
-<span class="sourceLineNo">090</span><a name="line.90"></a>
-<span class="sourceLineNo">091</span> public static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.91"></a>
-<span class="sourceLineNo">092</span> static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.92"></a>
-<span class="sourceLineNo">093</span><a name="line.93"></a>
-<span class="sourceLineNo">094</span> /**<a name="line.94"></a>
-<span class="sourceLineNo">095</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.95"></a>
-<span class="sourceLineNo">096</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.96"></a>
-<span class="sourceLineNo">097</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.97"></a>
-<span class="sourceLineNo">098</span> */<a name="line.98"></a>
-<span class="sourceLineNo">099</span> volatile Testing testing = null;<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> /**<a name="line.101"></a>
-<span class="sourceLineNo">102</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.102"></a>
-<span class="sourceLineNo">103</span> */<a name="line.103"></a>
-<span class="sourceLineNo">104</span> public static class Testing {<a name="line.104"></a>
-<span class="sourceLineNo">105</span> protected volatile boolean killIfHasParent = true;<a name="line.105"></a>
-<span class="sourceLineNo">106</span> protected volatile boolean killIfSuspended = false;<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span> /**<a name="line.108"></a>
-<span class="sourceLineNo">109</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.109"></a>
-<span class="sourceLineNo">110</span> * persisting all the state it needs to recover after a crash.<a name="line.110"></a>
-<span class="sourceLineNo">111</span> */<a name="line.111"></a>
-<span class="sourceLineNo">112</span> protected volatile boolean killBeforeStoreUpdate = false;<a name="line.112"></a>
-<span class="sourceLineNo">113</span> protected volatile boolean toggleKillBeforeStoreUpdate = false;<a name="line.113"></a>
-<span class="sourceLineNo">114</span><a name="line.114"></a>
-<span class="sourceLineNo">115</span> /**<a name="line.115"></a>
-<span class="sourceLineNo">116</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.116"></a>
-<span class="sourceLineNo">117</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.117"></a>
-<span class="sourceLineNo">118</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.118"></a>
-<span class="sourceLineNo">119</span> */<a name="line.119"></a>
-<span class="sourceLineNo">120</span> protected volatile boolean killAfterStoreUpdate = false;<a name="line.120"></a>
-<span class="sourceLineNo">121</span> protected volatile boolean toggleKillAfterStoreUpdate = false;<a name="line.121"></a>
-<span class="sourceLineNo">122</span><a name="line.122"></a>
-<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.123"></a>
-<span class="sourceLineNo">124</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.124"></a>
-<span class="sourceLineNo">125</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> this.killBeforeStoreUpdate = !kill;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.127"></a>
-<span class="sourceLineNo">128</span> }<a name="line.128"></a>
-<span class="sourceLineNo">129</span> return kill;<a name="line.129"></a>
-<span class="sourceLineNo">130</span> }<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span> protected boolean shouldKillBeforeStoreUpdate(boolean isSuspended, boolean hasParent) {<a name="line.132"></a>
-<span class="sourceLineNo">133</span> if (isSuspended && !killIfSuspended) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> return false;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span> if (hasParent && !killIfHasParent) {<a name="line.136"></a>
-<span class="sourceLineNo">137</span> return false;<a name="line.137"></a>
-<span class="sourceLineNo">138</span> }<a name="line.138"></a>
-<span class="sourceLineNo">139</span> return shouldKillBeforeStoreUpdate();<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.142"></a>
-<span class="sourceLineNo">143</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.143"></a>
-<span class="sourceLineNo">144</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.144"></a>
-<span class="sourceLineNo">145</span> this.killAfterStoreUpdate = !kill;<a name="line.145"></a>
-<span class="sourceLineNo">146</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.146"></a>
-<span class="sourceLineNo">147</span> }<a name="line.147"></a>
-<span class="sourceLineNo">148</span> return kill;<a name="line.148"></a>
-<span class="sourceLineNo">149</span> }<a name="line.149"></a>
-<span class="sourceLineNo">150</span><a name="line.150"></a>
-<span class="sourceLineNo">151</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.151"></a>
-<span class="sourceLineNo">152</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.152"></a>
-<span class="sourceLineNo">153</span> }<a name="line.153"></a>
-<span class="sourceLineNo">154</span> }<a name="line.154"></a>
-<span class="sourceLineNo">155</span><a name="line.155"></a>
-<span class="sourceLineNo">156</span> public interface ProcedureExecutorListener {<a name="line.156"></a>
-<span class="sourceLineNo">157</span> void procedureLoaded(long procId);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> void procedureAdded(long procId);<a name="line.158"></a>
-<span class="sourceLineNo">159</span> void procedureFinished(long procId);<a name="line.159"></a>
-<span class="sourceLineNo">160</span> }<a name="line.160"></a>
-<span class="sourceLineNo">161</span><a name="line.161"></a>
-<span class="sourceLineNo">162</span> /**<a name="line.162"></a>
-<span class="sourceLineNo">163</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.163"></a>
-<span class="sourceLineNo">164</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.164"></a>
-<span class="sourceLineNo">165</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.165"></a>
-<span class="sourceLineNo">166</span> */<a name="line.166"></a>
-<span class="sourceLineNo">167</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.167"></a>
-<span class="sourceLineNo">168</span> new ConcurrentHashMap<>();<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> /**<a name="line.170"></a>
-<span class="sourceLineNo">171</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.171"></a>
-<span class="sourceLineNo">172</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.172"></a>
-<span class="sourceLineNo">173</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.173"></a>
-<span class="sourceLineNo">174</span> */<a name="line.174"></a>
-<span class="sourceLineNo">175</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.175"></a>
-<span class="sourceLineNo">176</span> new ConcurrentHashMap<>();<a name="line.176"></a>
-<span class="sourceLineNo">177</span><a name="line.177"></a>
-<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
-<span class="sourceLineNo">179</span> * Helper map to lookup the live procedures by ID.<a name="line.179"></a>
-<span class="sourceLineNo">180</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.180"></a>
-<span class="sourceLineNo">181</span> */<a name="line.181"></a>
-<span class="sourceLineNo">182</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.182"></a>
-<span class="sourceLineNo">183</span> new ConcurrentHashMap<>();<a name="line.183"></a>
-<span class="sourceLineNo">184</span><a name="line.184"></a>
-<span class="sourceLineNo">185</span> /**<a name="line.185"></a>
-<span class="sourceLineNo">186</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.186"></a>
-<span class="sourceLineNo">187</span> * contains every root procedure.<a name="line.187"></a>
-<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.191"></a>
-<span class="sourceLineNo">192</span> new CopyOnWriteArrayList<>();<a name="line.192"></a>
-<span class="sourceLineNo">193</span><a name="line.193"></a>
-<span class="sourceLineNo">194</span> private Configuration conf;<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span> /**<a name="line.196"></a>
-<span class="sourceLineNo">197</span> * Created in the {@link #init(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.197"></a>
-<span class="sourceLineNo">198</span> * resource handling rather than observing in a #join is unexpected).<a name="line.198"></a>
-<span class="sourceLineNo">199</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.199"></a>
-<span class="sourceLineNo">200</span> * (Should be ok).<a name="line.200"></a>
-<span class="sourceLineNo">201</span> */<a name="line.201"></a>
-<span class="sourceLineNo">202</span> private ThreadGroup threadGroup;<a name="line.202"></a>
-<span class="sourceLineNo">203</span><a name="line.203"></a>
-<span class="sourceLineNo">204</span> /**<a name="line.204"></a>
-<span class="sourceLineNo">205</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.205"></a>
-<span class="sourceLineNo">206</span> * resource handling rather than observing in a #join is unexpected).<a name="line.206"></a>
-<span class="sourceLineNo">207</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.207"></a>
-<span class="sourceLineNo">208</span> * (Should be ok).<a name="line.208"></a>
-<span class="sourceLineNo">209</span> */<a name="line.209"></a>
-<span class="sourceLineNo">210</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.210"></a>
-<span class="sourceLineNo">211</span><a name="line.211"></a>
-<span class="sourceLineNo">212</span> /**<a name="line.212"></a>
-<span class="sourceLineNo">213</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.213"></a>
-<span class="sourceLineNo">214</span> * resource handling rather than observing in a #join is unexpected).<a name="line.214"></a>
-<span class="sourceLineNo">215</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.215"></a>
-<span class="sourceLineNo">216</span> * (Should be ok).<a name="line.216"></a>
-<span class="sourceLineNo">217</span> */<a name="line.217"></a>
-<span class="sourceLineNo">218</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> /**<a name="line.220"></a>
-<span class="sourceLineNo">221</span> * WorkerMonitor check for stuck workers and new worker thread when necessary, for example if<a name="line.221"></a>
-<span class="sourceLineNo">222</span> * there is no worker to assign meta, it will new worker thread for it, so it is very important.<a name="line.222"></a>
-<span class="sourceLineNo">223</span> * TimeoutExecutor execute many tasks like DeadServerMetricRegionChore RegionInTransitionChore<a name="line.223"></a>
-<span class="sourceLineNo">224</span> * and so on, some tasks may execute for a long time so will block other tasks like<a name="line.224"></a>
-<span class="sourceLineNo">225</span> * WorkerMonitor, so use a dedicated thread for executing WorkerMonitor.<a name="line.225"></a>
-<span class="sourceLineNo">226</span> */<a name="line.226"></a>
-<span class="sourceLineNo">227</span> private TimeoutExecutorThread<TEnvironment> workerMonitorExecutor;<a name="line.227"></a>
-<span class="sourceLineNo">228</span><a name="line.228"></a>
-<span class="sourceLineNo">229</span> private int corePoolSize;<a name="line.229"></a>
-<span class="sourceLineNo">230</span> private int maxPoolSize;<a name="line.230"></a>
-<span class="sourceLineNo">231</span><a name="line.231"></a>
-<span class="sourceLineNo">232</span> private volatile long keepAliveTime;<a name="line.232"></a>
-<span class="sourceLineNo">233</span><a name="line.233"></a>
-<span class="sourceLineNo">234</span> /**<a name="line.234"></a>
-<span class="sourceLineNo">235</span> * Scheduler/Queue that contains runnable procedures.<a name="line.235"></a>
-<span class="sourceLineNo">236</span> */<a name="line.236"></a>
-<span class="sourceLineNo">237</span> private final ProcedureScheduler scheduler;<a name="line.237"></a>
-<span class="sourceLineNo">238</span><a name="line.238"></a>
-<span class="sourceLineNo">239</span> private final Executor forceUpdateExecutor = Executors.newSingleThreadExecutor(<a name="line.239"></a>
-<span class="sourceLineNo">240</span> new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());<a name="line.240"></a>
-<span class="sourceLineNo">241</span><a name="line.241"></a>
-<span class="sourceLineNo">242</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.242"></a>
-<span class="sourceLineNo">243</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.243"></a>
-<span class="sourceLineNo">244</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.245"></a>
-<span class="sourceLineNo">246</span> private final TEnvironment environment;<a name="line.246"></a>
-<span class="sourceLineNo">247</span> private final ProcedureStore store;<a name="line.247"></a>
-<span class="sourceLineNo">248</span><a name="line.248"></a>
-<span class="sourceLineNo">249</span> private final boolean checkOwnerSet;<a name="line.249"></a>
-<span class="sourceLineNo">250</span><a name="line.250"></a>
-<span class="sourceLineNo">251</span> // To prevent concurrent execution of the same procedure.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.252"></a>
-<span class="sourceLineNo">253</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.253"></a>
-<span class="sourceLineNo">254</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.254"></a>
-<span class="sourceLineNo">255</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.255"></a>
-<span class="sourceLineNo">256</span> // execution of the same procedure.<a name="line.256"></a>
-<span class="sourceLineNo">257</span> private final IdLock procExecutionLock = new IdLock();<a name="line.257"></a>
-<span class="sourceLineNo">258</span><a name="line.258"></a>
-<span class="sourceLineNo">259</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.259"></a>
-<span class="sourceLineNo">260</span> final ProcedureStore store) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span><a name="line.263"></a>
-<span class="sourceLineNo">264</span> private boolean isRootFinished(Procedure<?> proc) {<a name="line.264"></a>
-<span class="sourceLineNo">265</span> Procedure<?> rootProc = procedures.get(proc.getRootProcId());<a name="line.265"></a>
-<span class="sourceLineNo">266</span> return rootProc == null || rootProc.isFinished();<a name="line.266"></a>
-<span class="sourceLineNo">267</span> }<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private void forceUpdateProcedure(long procId) throws IOException {<a name="line.269"></a>
-<span class="sourceLineNo">270</span> IdLock.Entry lockEntry = procExecutionLock.getLockEntry(procId);<a name="line.270"></a>
-<span class="sourceLineNo">271</span> try {<a name="line.271"></a>
-<span class="sourceLineNo">272</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.272"></a>
-<span class="sourceLineNo">273</span> if (proc != null) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span> if (proc.isFinished() && proc.hasParent() && isRootFinished(proc)) {<a name="line.274"></a>
-<span class="sourceLineNo">275</span> LOG.debug("Procedure {} has already been finished and parent is succeeded," +<a name="line.275"></a>
-<span class="sourceLineNo">276</span> " skip force updating", proc);<a name="line.276"></a>
-<span class="sourceLineNo">277</span> return;<a name="line.277"></a>
-<span class="sourceLineNo">278</span> }<a name="line.278"></a>
-<span class="sourceLineNo">279</span> } else {<a name="line.279"></a>
-<span class="sourceLineNo">280</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.280"></a>
-<span class="sourceLineNo">281</span> if (retainer == null || retainer.getProcedure() instanceof FailedProcedure) {<a name="line.281"></a>
-<span class="sourceLineNo">282</span> LOG.debug("No pending procedure with id = {}, skip force updating.", procId);<a name="line.282"></a>
-<span class="sourceLineNo">283</span> return;<a name="line.283"></a>
-<span class="sourceLineNo">284</span> }<a name="line.284"></a>
-<span class="sourceLineNo">285</span> long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.285"></a>
-<span class="sourceLineNo">286</span> long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.286"></a>
-<span class="sourceLineNo">287</span> if (retainer.isExpired(System.currentTimeMillis(), evictTtl, evictAckTtl)) {<a name="line.287"></a>
-<span class="sourceLineNo">288</span> LOG.debug("Procedure {} has already been finished and expired, skip force updating",<a name="line.288"></a>
-<span class="sourceLineNo">289</span> procId);<a name="line.289"></a>
-<span class="sourceLineNo">290</span> return;<a name="line.290"></a>
-<span class="sourceLineNo">291</span> }<a name="line.291"></a>
-<span class="sourceLineNo">292</span> proc = retainer.getProcedure();<a name="line.292"></a>
-<span class="sourceLineNo">293</span> }<a name="line.293"></a>
-<span class="sourceLineNo">294</span> LOG.debug("Force update procedure {}", proc);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> store.update(proc);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> } finally {<a name="line.296"></a>
-<span class="sourceLineNo">297</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.297"></a>
-<span class="sourceLineNo">298</span> }<a name="line.298"></a>
-<span class="sourceLineNo">299</span> }<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.301"></a>
-<span class="sourceLineNo">302</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.302"></a>
-<span class="sourceLineNo">303</span> this.environment = environment;<a name="line.303"></a>
-<span class="sourceLineNo">304</span> this.scheduler = scheduler;<a name="line.304"></a>
-<span class="sourceLineNo">305</span> this.store = store;<a name="line.305"></a>
-<span class="sourceLineNo">306</span> this.conf = conf;<a name="line.306"></a>
-<span class="sourceLineNo">307</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.307"></a>
-<span class="sourceLineNo">308</span> refreshConfiguration(conf);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> store.registerListener(new ProcedureStoreListener() {<a name="line.309"></a>
-<span class="sourceLineNo">310</span><a name="line.310"></a>
-<span class="sourceLineNo">311</span> @Override<a name="line.311"></a>
-<span class="sourceLineNo">312</span> public void forceUpdate(long[] procIds) {<a name="line.312"></a>
-<span class="sourceLineNo">313</span> Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> {<a name="line.313"></a>
-<span class="sourceLineNo">314</span> try {<a name="line.314"></a>
-<span class="sourceLineNo">315</span> forceUpdateProcedure(procId);<a name="line.315"></a>
-<span class="sourceLineNo">316</span> } catch (IOException e) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span> LOG.warn("Failed to force update procedure with pid={}", procId);<a name="line.317"></a>
-<span class="sourceLineNo">318</span> }<a name="line.318"></a>
-<span class="sourceLineNo">319</span> }));<a name="line.319"></a>
-<span class="sourceLineNo">320</span> }<a name="line.320"></a>
-<span class="sourceLineNo">321</span> });<a name="line.321"></a>
-<span class="sourceLineNo">322</span> }<a name="line.322"></a>
-<span class="sourceLineNo">323</span><a name="line.323"></a>
-<span class="sourceLineNo">324</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.325"></a>
-<span class="sourceLineNo">326</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.326"></a>
-<span class="sourceLineNo">327</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.327"></a>
-<span class="sourceLineNo">328</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.330"></a>
-<span class="sourceLineNo">331</span> @Override<a name="line.331"></a>
-<span class="sourceLineNo">332</span> public void setMaxProcId(long maxProcId) {<a name="line.332"></a>
-<span class="sourceLineNo">333</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.333"></a>
-<span class="sourceLineNo">334</span> lastProcId.set(maxProcId);<a name="line.334"></a>
-<span class="sourceLineNo">335</span> }<a name="line.335"></a>
-<span class="sourceLineNo">336</span><a name="line.336"></a>
-<span class="sourceLineNo">337</span> @Override<a name="line.337"></a>
-<span class="sourceLineNo">338</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.338"></a>
-<span class="sourceLineNo">339</span> loadProcedures(procIter, abortOnCorruption);<a name="line.339"></a>
-<span class="sourceLineNo">340</span> }<a name="line.340"></a>
-<span class="sourceLineNo">341</span><a name="line.341"></a>
-<span class="sourceLineNo">342</span> @Override<a name="line.342"></a>
-<span class="sourceLineNo">343</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.343"></a>
-<span class="sourceLineNo">344</span> int corruptedCount = 0;<a name="line.344"></a>
-<span class="sourceLineNo">345</span> while (procIter.hasNext()) {<a name="line.345"></a>
-<span class="sourceLineNo">346</span> Procedure<?> proc = procIter.next();<a name="line.346"></a>
-<span class="sourceLineNo">347</span> LOG.error("Corrupt " + proc);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> corruptedCount++;<a name="line.348"></a>
-<span class="sourceLineNo">349</span> }<a name="line.349"></a>
-<span class="sourceLineNo">350</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.350"></a>
-<span class="sourceLineNo">351</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span> }<a name="line.353"></a>
-<span class="sourceLineNo">354</span> });<a name="line.354"></a>
-<span class="sourceLineNo">355</span> }<a name="line.355"></a>
-<span class="sourceLineNo">356</span><a name="line.356"></a>
-<span class="sourceLineNo">357</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.357"></a>
-<span class="sourceLineNo">358</span> proc.restoreLock(getEnvironment());<a name="line.358"></a>
-<span class="sourceLineNo">359</span> restored.add(proc.getProcId());<a name="line.359"></a>
-<span class="sourceLineNo">360</span> }<a name="line.360"></a>
-<span class="sourceLineNo">361</span><a name="line.361"></a>
-<span class="sourceLineNo">362</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> while (!stack.isEmpty()) {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> restoreLock(stack.pop(), restored);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span><a name="line.367"></a>
-<span class="sourceLineNo">368</span> // Restore the locks for all the procedures.<a name="line.368"></a>
-<span class="sourceLineNo">369</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.369"></a>
-<span class="sourceLineNo">370</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.370"></a>
-<span class="sourceLineNo">371</span> // calling the acquireLock method for the parent procedure.<a name="line.371"></a>
-<span class="sourceLineNo">372</span> // The algorithm is straight-forward:<a name="line.372"></a>
-<span class="sourceLineNo">373</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.373"></a>
-<span class="sourceLineNo">374</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.374"></a>
-<span class="sourceLineNo">375</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.375"></a>
-<span class="sourceLineNo">376</span> // unless<a name="line.376"></a>
-<span class="sourceLineNo">377</span> // a. We have no parent, i.e, we are the root procedure<a name="line.377"></a>
-<span class="sourceLineNo">378</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.378"></a>
-<span class="sourceLineNo">379</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.379"></a>
-<span class="sourceLineNo">380</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.380"></a>
-<span class="sourceLineNo">381</span> private void restoreLocks() {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> Set<Long> restored = new HashSet<>();<a name="line.382"></a>
-<span class="sourceLineNo">383</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.383"></a>
-<span class="sourceLineNo">384</span> procedures.values().forEach(proc -> {<a name="line.384"></a>
-<span class="sourceLineNo">385</span> for (;;) {<a name="line.385"></a>
-<span class="sourceLineNo">386</span> if (restored.contains(proc.getProcId())) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span> restoreLocks(stack, restored);<a name="line.387"></a>
-<span class="sourceLineNo">388</span> return;<a name="line.388"></a>
-<span class="sourceLineNo">389</span> }<a name="line.389"></a>
-<span class="sourceLineNo">390</span> if (!proc.hasParent()) {<a name="line.390"></a>
-<span class="sourceLineNo">391</span> restoreLock(proc, restored);<a name="line.391"></a>
-<span class="sourceLineNo">392</span> restoreLocks(stack, restored);<a name="line.392"></a>
-<span class="sourceLineNo">393</span> return;<a name="line.393"></a>
-<span class="sourceLineNo">394</span> }<a name="line.394"></a>
-<span class="sourceLineNo">395</span> stack.push(proc);<a name="line.395"></a>
-<span class="sourceLineNo">396</span> proc = procedures.get(proc.getParentProcId());<a name="line.396"></a>
-<span class="sourceLineNo">397</span> }<a name="line.397"></a>
-<span class="sourceLineNo">398</span> });<a name="line.398"></a>
-<span class="sourceLineNo">399</span> }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.401"></a>
-<span class="sourceLineNo">402</span> throws IOException {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> // 1. Build the rollback stack<a name="line.403"></a>
-<span class="sourceLineNo">404</span> int runnableCount = 0;<a name="line.404"></a>
-<span class="sourceLineNo">405</span> int failedCount = 0;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> int waitingCount = 0;<a name="line.406"></a>
-<span class="sourceLineNo">407</span> int waitingTimeoutCount = 0;<a name="line.407"></a>
-<span class="sourceLineNo">408</span> while (procIter.hasNext()) {<a name="line.408"></a>
-<span class="sourceLineNo">409</span> boolean finished = procIter.isNextFinished();<a name="line.409"></a>
-<span class="sourceLineNo">410</span> @SuppressWarnings("unchecked")<a name="line.410"></a>
-<span class="sourceLineNo">411</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.411"></a>
-<span class="sourceLineNo">412</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.412"></a>
-<span class="sourceLineNo">413</span> long procId = proc.getProcId();<a name="line.413"></a>
-<span class="sourceLineNo">414</span><a name="line.414"></a>
-<span class="sourceLineNo">415</span> if (finished) {<a name="line.415"></a>
-<span class="sourceLineNo">416</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.416"></a>
-<span class="sourceLineNo">417</span> LOG.debug("Completed {}", proc);<a name="line.417"></a>
-<span class="sourceLineNo">418</span> } else {<a name="line.418"></a>
-<span class="sourceLineNo">419</span> if (!proc.hasParent()) {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.420"></a>
-<span class="sourceLineNo">421</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.421"></a>
-<span class="sourceLineNo">422</span> }<a name="line.422"></a>
-<span class="sourceLineNo">423</span><a name="line.423"></a>
-<span class="sourceLineNo">424</span> // add the procedure to the map<a name="line.424"></a>
-<span class="sourceLineNo">425</span> proc.beforeReplay(getEnvironment());<a name="line.425"></a>
-<span class="sourceLineNo">426</span> procedures.put(proc.getProcId(), proc);<a name="line.426"></a>
-<span class="sourceLineNo">427</span> switch (proc.getState()) {<a name="line.427"></a>
-<span class="sourceLineNo">428</span> case RUNNABLE:<a name="line.428"></a>
-<span class="sourceLineNo">429</span> runnableCount++;<a name="line.429"></a>
-<span class="sourceLineNo">430</span> break;<a name="line.430"></a>
-<span class="sourceLineNo">431</span> case FAILED:<a name="line.431"></a>
-<span class="sourceLineNo">432</span> failedCount++;<a name="line.432"></a>
-<span class="sourceLineNo">433</span> break;<a name="line.433"></a>
-<span class="sourceLineNo">434</span> case WAITING:<a name="line.434"></a>
-<span class="sourceLineNo">435</span> waitingCount++;<a name="line.435"></a>
-<span class="sourceLineNo">436</span> break;<a name="line.436"></a>
-<span class="sourceLineNo">437</span> case WAITING_TIMEOUT:<a name="line.437"></a>
-<span class="sourceLineNo">438</span> waitingTimeoutCount++;<a name="line.438"></a>
-<span class="sourceLineNo">439</span> break;<a name="line.439"></a>
-<span class="sourceLineNo">440</span> default:<a name="line.440"></a>
-<span class="sourceLineNo">441</span> break;<a name="line.441"></a>
-<span class="sourceLineNo">442</span> }<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span><a name="line.444"></a>
-<span class="sourceLineNo">445</span> if (nonceKey != null) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span> nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the map<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span> }<a name="line.448"></a>
-<span class="sourceLineNo">449</span><a name="line.449"></a>
-<span class="sourceLineNo">450</span> // 2. Initialize the stacks: In the old implementation, for procedures in FAILED state, we will<a name="line.450"></a>
-<span class="sourceLineNo">451</span> // push it into the ProcedureScheduler directly to execute the rollback. But this does not work<a name="line.451"></a>
-<span class="sourceLineNo">452</span> // after we introduce the restore lock stage. For now, when we acquire a xlock, we will remove<a name="line.452"></a>
-<span class="sourceLineNo">453</span> // the queue from runQueue in scheduler, and then when a procedure which has lock access, for<a name="line.453"></a>
-<span class="sourceLineNo">454</span> // example, a sub procedure of the procedure which has the xlock, is pushed into the scheduler,<a name="line.454"></a>
-<span class="sourceLineNo">455</span> // we will add the queue back to let the workers poll from it. The assumption here is that, the<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // procedure which has the xlock should have been polled out already, so when loading we can not<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // add the procedure to scheduler first and then call acquireLock, since the procedure is still<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // in the queue, and since we will remove the queue from runQueue, then no one can poll it out,<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // then there is a dead lock<a name="line.459"></a>
-<span class="sourceLineNo">460</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnableCount);<a name="line.460"></a>
-<span class="sourceLineNo">461</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.461"></a>
-<span class="sourceLineNo">462</span> List<Procedure<TEnvironment>> waitingList = new ArrayList<>(waitingCount);<a name="line.462"></a>
-<span class="sourceLineNo">463</span> List<Procedure<TEnvironment>> waitingTimeoutList = new ArrayList<>(waitingTimeoutCount);<a name="line.463"></a>
-<span class="sourceLineNo">464</span> procIter.reset();<a name="line.464"></a>
-<span class="sourceLineNo">465</span> while (procIter.hasNext()) {<a name="line.465"></a>
-<span class="sourceLineNo">466</span> if (procIter.isNextFinished()) {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> procIter.skipNext();<a name="line.467"></a>
-<span class="sourceLineNo">468</span> continue;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> }<a name="line.469"></a>
-<span class="sourceLineNo">470</span><a name="line.470"></a>
-<span class="sourceLineNo">471</span> @SuppressWarnings("unchecked")<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> LOG.debug("Loading {}", proc);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Long rootProcId = getRootProcedureId(proc);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> // The orphan procedures will be passed to handleCorrupted, so add an assert here<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert rootProcId != null;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> if (proc.hasParent()) {<a name="line.479"></a>
-<span class="sourceLineNo">480</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.480"></a>
-<span class="sourceLineNo">481</span> if (parent != null && !proc.isFinished()) {<a name="line.481"></a>
-<span class="sourceLineNo">482</span> parent.incChildrenLatch();<a name="line.482"></a>
-<span class="sourceLineNo">483</span> }<a name="line.483"></a>
-<span class="sourceLineNo">484</span> }<a name="line.484"></a>
-<span class="sourceLineNo">485</span><a name="line.485"></a>
-<span class="sourceLineNo">486</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> procStack.loadStack(proc);<a name="line.487"></a>
-<span class="sourceLineNo">488</span><a name="line.488"></a>
-<span class="sourceLineNo">489</span> proc.setRootProcId(rootProcId);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> switch (proc.getState()) {<a name="line.490"></a>
-<span class="sourceLineNo">491</span> case RUNNABLE:<a name="line.491"></a>
-<span class="sourceLineNo">492</span> runnableList.add(proc);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> break;<a name="line.493"></a>
-<span class="sourceLineNo">494</span> case WAITING:<a name="line.494"></a>
-<span class="sourceLineNo">495</span> waitingList.add(proc);<a name="line.495"></a>
-<span class="sourceLineNo">496</span> break;<a name="line.496"></a>
-<span class="sourceLineNo">497</span> case WAITING_TIMEOUT:<a name="line.497"></a>
-<span class="sourceLineNo">498</span> waitingTimeoutList.add(proc);<a name="line.498"></a>
-<span class="sourceLineNo">499</span> break;<a name="line.499"></a>
-<span class="sourceLineNo">500</span> case FAILED:<a name="line.500"></a>
-<span class="sourceLineNo">501</span> failedList.add(proc);<a name="line.501"></a>
-<span class="sourceLineNo">502</span> break;<a name="line.502"></a>
-<span class="sourceLineNo">503</span> case ROLLEDBACK:<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case INITIALIZING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.505"></a>
-<span class="sourceLineNo">506</span> LOG.error(msg);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> throw new UnsupportedOperationException(msg);<a name="line.507"></a>
-<span class="sourceLineNo">508</span> default:<a name="line.508"></a>
-<span class="sourceLineNo">509</span> break;<a name="line.509"></a>
-<span class="sourceLineNo">510</span> }<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> // 3. Check the waiting procedures to see if some of them can be added to runnable.<a name="line.513"></a>
-<span class="sourceLineNo">514</span> waitingList.forEach(proc -> {<a name="line.514"></a>
-<span class="sourceLineNo">515</span> if (!proc.hasChildren()) {<a name="line.515"></a>
-<span class="sourceLineNo">516</span> // Normally, WAITING procedures should be waken by its children. But, there is a case that,<a name="line.516"></a>
-<span class="sourceLineNo">517</span> // all the children are successful and before they can wake up their parent procedure, the<a name="line.517"></a>
-<span class="sourceLineNo">518</span> // master was killed. So, during recovering the procedures from ProcedureWal, its children<a name="line.518"></a>
-<span class="sourceLineNo">519</span> // are not loaded because of their SUCCESS state. So we need to continue to run this WAITING<a name="line.519"></a>
-<span class="sourceLineNo">520</span> // procedure. But before executing, we need to set its state to RUNNABLE, otherwise, a<a name="line.520"></a>
-<span class="sourceLineNo">521</span> // exception will throw:<a name="line.521"></a>
-<span class="sourceLineNo">522</span> // Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.522"></a>
-<span class="sourceLineNo">523</span> // "NOT RUNNABLE! " + procedure.toString());<a name="line.523"></a>
-<span class="sourceLineNo">524</span> proc.setState(ProcedureState.RUNNABLE);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> runnableList.add(proc);<a name="line.525"></a>
-<span class="sourceLineNo">526</span> } else {<a name="line.526"></a>
-<span class="sourceLineNo">527</span> proc.afterReplay(getEnvironment());<a name="line.527"></a>
-<span class="sourceLineNo">528</span> }<a name="line.528"></a>
-<span class="sourceLineNo">529</span> });<a name="line.529"></a>
-<span class="sourceLineNo">530</span> // 4. restore locks<a name="line.530"></a>
-<span class="sourceLineNo">531</span> restoreLocks();<a name="line.531"></a>
-<span class="sourceLineNo">532</span><a name="line.532"></a>
-<span class="sourceLineNo">533</span> // 5. Push the procedures to the timeout executor<a name="line.533"></a>
-<span class="sourceLineNo">534</span> waitingTimeoutList.forEach(proc -> {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> proc.afterReplay(getEnvironment());<a name="line.535"></a>
-<span class="sourceLineNo">536</span> timeoutExecutor.add(proc);<a name="line.536"></a>
-<span class="sourceLineNo">537</span> });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span> // 6. Push the procedure to the scheduler<a name="line.539"></a>
-<span class="sourceLineNo">540</span> failedList.forEach(scheduler::addBack);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> runnableList.forEach(p -> {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> p.afterReplay(getEnvironment());<a name="line.542"></a>
-<span class="sourceLineNo">543</span> if (!p.hasParent()) {<a name="line.543"></a>
-<span class="sourceLineNo">544</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> scheduler.addBack(p);<a name="line.546"></a>
-<span class="sourceLineNo">547</span> });<a name="line.547"></a>
-<span class="sourceLineNo">548</span> // After all procedures put into the queue, signal the worker threads.<a name="line.548"></a>
-<span class="sourceLineNo">549</span> // Otherwise, there is a race condition. See HBASE-21364.<a name="line.549"></a>
-<span class="sourceLineNo">550</span> scheduler.signalAll();<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> /**<a name="line.553"></a>
-<span class="sourceLineNo">554</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.554"></a>
-<span class="sourceLineNo">555</span> * <p/><a name="line.555"></a>
-<span class="sourceLineNo">556</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.556"></a>
-<span class="sourceLineNo">557</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.557"></a>
-<span class="sourceLineNo">558</span> * pending and in-progress procedures.<a name="line.558"></a>
-<span class="sourceLineNo">559</span> * @param numThreads number of threads available for procedure execution.<a name="line.559"></a>
-<span class="sourceLineNo">560</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.560"></a>
-<span class="sourceLineNo">561</span> * is found on replay. otherwise false.<a name="line.561"></a>
-<span class="sourceLineNo">562</span> */<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> // We have numThreads executor + one timer thread used for timing out<a name="line.564"></a>
-<span class="sourceLineNo">565</span> // procedures and triggering periodic procedures.<a name="line.565"></a>
-<span class="sourceLineNo">566</span> this.corePoolSize = numThreads;<a name="line.566"></a>
-<span class="sourceLineNo">567</span> this.maxPoolSize = 10 * numThreads;<a name="line.567"></a>
-<span class="sourceLineNo">568</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.568"></a>
-<span class="sourceLineNo">569</span> corePoolSize, maxPoolSize);<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.571"></a>
-<span class="sourceLineNo">572</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, "ProcExecTimeout");<a name="line.572"></a>
-<span class="sourceLineNo">573</span> this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, threadGroup, "WorkerMonitor");<a name="line.573"></a>
-<span class="sourceLineNo">574</span><a name="line.574"></a>
-<span class="sourceLineNo">575</span> // Create the workers<a name="line.575"></a>
-<span class="sourceLineNo">576</span> workerId.set(0);<a name="line.576"></a>
-<span class="sourceLineNo">577</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.577"></a>
-<span class="sourceLineNo">578</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span> long st, et;<a name="line.582"></a>
-<span class="sourceLineNo">583</span><a name="line.583"></a>
-<span class="sourceLineNo">584</span> // Acquire the store lease.<a name="line.584"></a>
-<span class="sourceLineNo">585</span> st = System.nanoTime();<a name="line.585"></a>
-<span class="sourceLineNo">586</span> store.recoverLease();<a name="line.586"></a>
-<span class="sourceLineNo">587</span> et = System.nanoTime();<a name="line.587"></a>
-<span class="sourceLineNo">588</span> LOG.info("Recovered {} lease in {}", store.getClass().getSimpleName(),<a name="line.588"></a>
-<span class="sourceLineNo">589</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.589"></a>
-<span class="sourceLineNo">590</span><a name="line.590"></a>
-<span class="sourceLineNo">591</span> // start the procedure scheduler<a name="line.591"></a>
-<span class="sourceLineNo">592</span> scheduler.start();<a name="line.592"></a>
-<span class="sourceLineNo">593</span><a name="line.593"></a>
-<span class="sourceLineNo">594</span> // TODO: Split in two steps.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> // TODO: Handle corrupted procedures (currently just a warn)<a name="line.595"></a>
-<span class="sourceLineNo">596</span> // The first one will make sure that we have the latest id,<a name="line.596"></a>
-<span class="sourceLineNo">597</span> // so we can start the threads and accept new procedures.<a name="line.597"></a>
-<span class="sourceLineNo">598</span> // The second step will do the actual load of old procedures.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> st = System.nanoTime();<a name="line.599"></a>
-<span class="sourceLineNo">600</span> load(abortOnCorruption);<a name="line.600"></a>
-<span class="sourceLineNo">601</span> et = System.nanoTime();<a name="line.601"></a>
-<span class="sourceLineNo">602</span> LOG.info("Loaded {} in {}", store.getClass().getSimpleName(),<a name="line.602"></a>
-<span class="sourceLineNo">603</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.603"></a>
-<span class="sourceLineNo">604</span> }<a name="line.604"></a>
-<span class="sourceLineNo">605</span><a name="line.605"></a>
-<span class="sourceLineNo">606</span> /**<a name="line.606"></a>
-<span class="sourceLineNo">607</span> * Start the workers.<a name="line.607"></a>
-<span class="sourceLineNo">608</span> */<a name="line.608"></a>
-<span class="sourceLineNo">609</span> public void startWorkers() throws IOException {<a name="line.609"></a>
-<span class="sourceLineNo">610</span> if (!running.compareAndSet(false, true)) {<a name="line.610"></a>
-<span class="sourceLineNo">611</span> LOG.warn("Already running");<a name="line.611"></a>
-<span class="sourceLineNo">612</span> return;<a name="line.612"></a>
-<span class="sourceLineNo">613</span> }<a name="line.613"></a>
-<span class="sourceLineNo">614</span> // Start the executors. Here we must have the lastProcId set.<a name="line.614"></a>
-<span class="sourceLineNo">615</span> LOG.trace("Start workers {}", workerThreads.size());<a name="line.615"></a>
-<span class="sourceLineNo">616</span> timeoutExecutor.start();<a name="line.616"></a>
-<span class="sourceLineNo">617</span> workerMonitorExecutor.start();<a name="line.617"></a>
-<span class="sourceLineNo">618</span> for (WorkerThread worker: workerThreads) {<a name="line.618"></a>
-<span class="sourceLineNo">619</span> worker.start();<a name="line.619"></a>
-<span class="sourceLineNo">620</span> }<a name="line.620"></a>
-<span class="sourceLineNo">621</span><a name="line.621"></a>
-<span class="sourceLineNo">622</span> // Internal chores<a name="line.622"></a>
-<span class="sourceLineNo">623</span> workerMonitorExecutor.add(new WorkerMonitor());<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> // Add completed cleaner chore<a name="line.625"></a>
-<span class="sourceLineNo">626</span> addChore(new CompletedProcedureCleaner<>(conf, store, procExecutionLock, completed,<a name="line.626"></a>
-<span class="sourceLineNo">627</span> nonceKeysToProcIdsMap));<a name="line.627"></a>
-<span class="sourceLineNo">628</span> }<a name="line.628"></a>
-<span class="sourceLineNo">629</span><a name="line.629"></a>
-<span class="sourceLineNo">630</span> public void stop() {<a name="line.630"></a>
-<span class="sourceLineNo">631</span> if (!running.getAndSet(false)) {<a name="line.631"></a>
-<span class="sourceLineNo">632</span> return;<a name="line.632"></a>
-<span class="sourceLineNo">633</span> }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span> LOG.info("Stopping");<a name="line.635"></a>
-<span class="sourceLineNo">636</span> scheduler.stop();<a name="line.636"></a>
-<span class="sourceLineNo">637</span> timeoutExecutor.sendStopSignal();<a name="line.637"></a>
-<span class="sourceLineNo">638</span> workerMonitorExecutor.sendStopSignal();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> }<a name="line.639"></a>
-<span class="sourceLineNo">640</span><a name="line.640"></a>
-<span class="sourceLineNo">641</span> @VisibleForTesting<a name="line.641"></a>
-<span class="sourceLineNo">642</span> public void join() {<a name="line.642"></a>
-<span class="sourceLineNo">643</span> assert !isRunning() : "expected not running";<a name="line.643"></a>
-<span class="sourceLineNo">644</span><a name="line.644"></a>
-<span class="sourceLineNo">645</span> // stop the timeout executor<a name="line.645"></a>
-<span class="sourceLineNo">646</span> timeoutExecutor.awaitTermination();<a name="line.646"></a>
-<span class="sourceLineNo">647</span> // stop the work monitor executor<a name="line.647"></a>
-<span class="sourceLineNo">648</span> workerMonitorExecutor.awaitTermination();<a name="line.648"></a>
-<span class="sourceLineNo">649</span><a name="line.649"></a>
-<span class="sourceLineNo">650</span> // stop the worker threads<a name="line.650"></a>
-<span class="sourceLineNo">651</span> for (WorkerThread worker: workerThreads) {<a name="line.651"></a>
-<span class="sourceLineNo">652</span> worker.awaitTermination();<a name="line.652"></a>
-<span class="sourceLineNo">653</span> }<a name="line.653"></a>
-<span class="sourceLineNo">654</span><a name="line.654"></a>
-<span class="sourceLineNo">655</span> // Destroy the Thread Group for the executors<a name="line.655"></a>
-<span class="sourceLineNo">656</span> // TODO: Fix. #join is not place to destroy resources.<a name="line.656"></a>
-<span class="sourceLineNo">657</span> try {<a name="line.657"></a>
-<span class="sourceLineNo">658</span> threadGroup.destroy();<a name="line.658"></a>
-<span class="sourceLineNo">659</span> } catch (IllegalThreadStateException e) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span> LOG.error("ThreadGroup {} contains running threads; {}: See STDOUT",<a name="line.660"></a>
-<span class="sourceLineNo">661</span> this.threadGroup, e.getMessage());<a name="line.661"></a>
-<span class="sourceLineNo">662</span> // This dumps list of threads on STDOUT.<a name="line.662"></a>
-<span class="sourceLineNo">663</span> this.threadGroup.list();<a name="line.663"></a>
-<span class="sourceLineNo">664</span> }<a name="line.664"></a>
-<span class="sourceLineNo">665</span><a name="line.665"></a>
-<span class="sourceLineNo">666</span> // reset the in-memory state for testing<a name="line.666"></a>
-<span class="sourceLineNo">667</span> completed.clear();<a name="line.667"></a>
-<span class="sourceLineNo">668</span> rollbackStack.clear();<a name="line.668"></a>
-<span class="sourceLineNo">669</span> procedures.clear();<a name="line.669"></a>
-<span class="sourceLineNo">670</span> nonceKeysToProcIdsMap.clear();<a name="line.670"></a>
-<span class="sourceLineNo">671</span> scheduler.clear();<a name="line.671"></a>
-<span class="sourceLineNo">672</span> lastProcId.set(-1);<a name="line.672"></a>
-<span class="sourceLineNo">673</span> }<a name="line.673"></a>
-<span class="sourceLineNo">674</span><a name="line.674"></a>
-<span class="sourceLineNo">675</span> public void refreshConfiguration(final Configuration conf) {<a name="line.675"></a>
-<span class="sourceLineNo">676</span> this.conf = conf;<a name="line.676"></a>
-<span class="sourceLineNo">677</span> setKeepAliveTime(conf.getLong(WORKER_KEEP_ALIVE_TIME_CONF_KEY,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> DEFAULT_WORKER_KEEP_ALIVE_TIME), TimeUnit.MILLISECONDS);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> }<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span> // ==========================================================================<a name="line.681"></a>
-<span class="sourceLineNo">682</span> // Accessors<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // ==========================================================================<a name="line.683"></a>
-<span class="sourceLineNo">684</span> public boolean isRunning() {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> return running.get();<a name="line.685"></a>
-<span class="sourceLineNo">686</span> }<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span> /**<a name="line.688"></a>
-<span class="sourceLineNo">689</span> * @return the current number of worker threads.<a name="line.689"></a>
-<span class="sourceLineNo">690</span> */<a name="line.690"></a>
-<span class="sourceLineNo">691</span> public int getWorkerThreadCount() {<a name="line.691"></a>
-<span class="sourceLineNo">692</span> return workerThreads.size();<a name="line.692"></a>
-<span class="sourceLineNo">693</span> }<a name="line.693"></a>
-<span class="sourceLineNo">694</span><a name="line.694"></a>
-<span class="sourceLineNo">695</span> /**<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @return the core pool size settings.<a name="line.696"></a>
-<span class="sourceLineNo">697</span> */<a name="line.697"></a>
-<span class="sourceLineNo">698</span> public int getCorePoolSize() {<a name="line.698"></a>
-<span class="sourceLineNo">699</span> return corePoolSize;<a name="line.699"></a>
-<span class="sourceLineNo">700</span> }<a name="line.700"></a>
-<span class="sourceLineNo">701</span><a name="line.701"></a>
-<span class="sourceLineNo">702</span> public int getActiveExecutorCount() {<a name="line.702"></a>
-<span class="sourceLineNo">703</span> return activeExecutorCount.get();<a name="line.703"></a>
-<span class="sourceLineNo">704</span> }<a name="line.704"></a>
-<span class="sourceLineNo">705</span><a name="line.705"></a>
-<span class="sourceLineNo">706</span> public TEnvironment getEnvironment() {<a name="line.706"></a>
-<span class="sourceLineNo">707</span> return this.environment;<a name="line.707"></a>
-<span class="sourceLineNo">708</span> }<a name="line.708"></a>
-<span class="sourceLineNo">709</span><a name="line.709"></a>
-<span class="sourceLineNo">710</span> public ProcedureStore getStore() {<a name="line.710"></a>
-<span class="sourceLineNo">711</span> return this.store;<a name="line.711"></a>
-<span class="sourceLineNo">712</span> }<a name="line.712"></a>
-<span class="sourceLineNo">713</span><a name="line.713"></a>
-<span class="sourceLineNo">714</span> ProcedureScheduler getScheduler() {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> return scheduler;<a name="line.715"></a>
-<span class="sourceLineNo">716</span> }<a name="line.716"></a>
-<span class="sourceLineNo">717</span><a name="line.717"></a>
-<span class="sourceLineNo">718</span> public void setKeepAliveTime(final long keepAliveTime, final TimeUnit timeUnit) {<a name="line.718"></a>
-<span class="sourceLineNo">719</span> this.keepAliveTime = timeUnit.toMillis(keepAliveTime);<a name="line.719"></a>
-<span class="sourceLineNo">720</span> this.scheduler.signalAll();<a name="line.720"></a>
-<span class="sourceLineNo">721</span> }<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span> public long getKeepAliveTime(final TimeUnit timeUnit) {<a name="line.723"></a>
-<span class="sourceLineNo">724</span> return timeUnit.convert(keepAliveTime, TimeUnit.MILLISECONDS);<a name="line.724"></a>
-<span class="sourceLineNo">725</span> }<a name="line.725"></a>
-<span class="sourceLineNo">726</span><a name="line.726"></a>
-<span class="sourceLineNo">727</span> // ==========================================================================<a name="line.727"></a>
-<span class="sourceLineNo">728</span> // Submit/Remove Chores<a name="line.728"></a>
-<span class="sourceLineNo">729</span> // ==========================================================================<a name="line.729"></a>
-<span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span> /**<a name="line.731"></a>
-<span class="sourceLineNo">732</span> * Add a chore procedure to the executor<a name="line.732"></a>
-<span class="sourceLineNo">733</span> * @param chore the chore to add<a name="line.733"></a>
-<span class="sourceLineNo">734</span> */<a name="line.734"></a>
-<span class="sourceLineNo">735</span> public void addChore(ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.735"></a>
-<span class="sourceLineNo">736</span> chore.setState(ProcedureState.WAITING_TIMEOUT);<a name="line.736"></a>
-<span class="sourceLineNo">737</span> timeoutExecutor.add(chore);<a name="line.737"></a>
-<span class="sourceLineNo">738</span> }<a name="line.738"></a>
-<span class="sourceLineNo">739</span><a name="line.739"></a>
-<span class="sourceLineNo">740</span> /**<a name="line.740"></a>
-<span class="sourceLineNo">741</span> * Remove a chore procedure from the executor<a name="line.741"></a>
-<span class="sourceLineNo">742</span> * @param chore the chore to remove<a name="line.742"></a>
-<span class="sourceLineNo">743</span> * @return whether the chore is removed, or it will be removed later<a name="line.743"></a>
-<span class="sourceLineNo">744</span> */<a name="line.744"></a>
-<span class="sourceLineNo">745</span> public boolean removeChore(ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.745"></a>
-<span class="sourceLineNo">746</span> chore.setState(ProcedureState.SUCCESS);<a name="line.746"></a>
-<span class="sourceLineNo">747</span> return timeoutExecutor.remove(chore);<a name="line.747"></a>
-<span class="sourceLineNo">748</span> }<a name="line.748"></a>
-<span class="sourceLineNo">749</span><a name="line.749"></a>
-<span class="sourceLineNo">750</span> // ==========================================================================<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // Nonce Procedure helpers<a name="line.751"></a>
-<span class="sourceLineNo">752</span> // ==========================================================================<a name="line.752"></a>
-<span class="sourceLineNo">753</span> /**<a name="line.753"></a>
-<span class="sourceLineNo">754</span> * Create a NonceKey from the specified nonceGroup and nonce.<a name="line.754"></a>
-<span class="sourceLineNo">755</span> * @param nonceGroup the group to use for the {@link NonceKey}<a name="line.755"></a>
-<span class="sourceLineNo">756</span> * @param nonce the nonce to use in the {@link NonceKey}<a name="line.756"></a>
-<span class="sourceLineNo">757</span> * @return the generated NonceKey<a name="line.757"></a>
-<span class="sourceLineNo">758</span> */<a name="line.758"></a>
-<span class="sourceLineNo">759</span> public NonceKey createNonceKey(final long nonceGroup, final long nonce) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> return (nonce == HConstants.NO_NONCE) ? null : new NonceKey(nonceGroup, nonce);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> }<a name="line.761"></a>
-<span class="sourceLineNo">762</span><a name="line.762"></a>
-<span class="sourceLineNo">763</span> /**<a name="line.763"></a>
-<span class="sourceLineNo">764</span> * Register a nonce for a procedure that is going to be submitted.<a name="line.764"></a>
-<span class="sourceLineNo">765</span> * A procId will be reserved and on submitProcedure(),<a name="line.765"></a>
-<span class="sourceLineNo">766</span> * the procedure with the specified nonce will take the reserved ProcId.<a name="line.766"></a>
-<span class="sourceLineNo">767</span> * If someone already reserved the nonce, this method will return the procId reserved,<a name="line.767"></a>
-<span class="sourceLineNo">768</span> * otherwise an invalid procId will be returned. and the caller should procede<a name="line.768"></a>
-<span class="sourceLineNo">769</span> * and submit the procedure.<a name="line.769"></a>
-<span class="sourceLineNo">770</span> *<a name="line.770"></a>
-<span class="sourceLineNo">771</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.771"></a>
-<span class="sourceLineNo">772</span> * @return the procId associated with the nonce, if any otherwise an invalid procId.<a name="line.772"></a>
-<span class="sourceLineNo">773</span> */<a name="line.773"></a>
-<span class="sourceLineNo">774</span> public long registerNonce(final NonceKey nonceKey) {<a name="line.774"></a>
-<span class="sourceLineNo">775</span> if (nonceKey == null) {<a name="line.775"></a>
-<span class="sourceLineNo">776</span> return -1;<a name="line.776"></a>
-<span class="sourceLineNo">777</span> }<a name="line.777"></a>
-<span class="sourceLineNo">778</span><a name="line.778"></a>
-<span class="sourceLineNo">779</span> // check if we have already a Reserved ID for the nonce<a name="line.779"></a>
-<span class="sourceLineNo">780</span> Long oldProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> if (oldProcId == null) {<a name="line.781"></a>
-<span class="sourceLineNo">782</span> // reserve a new Procedure ID, this will be associated with the nonce<a name="line.782"></a>
-<span class="sourceLineNo">783</span> // and the procedure submitted with the specified nonce will use this ID.<a name="line.783"></a>
-<span class="sourceLineNo">784</span> final long newProcId = nextProcId();<a name="line.784"></a>
-<span class="sourceLineNo">785</span> oldProcId = nonceKeysToProcIdsMap.putIfAbsent(nonceKey, newProcId);<a name="line.785"></a>
-<span class="sourceLineNo">786</span> if (oldProcId == null) {<a name="line.786"></a>
-<span class="sourceLineNo">787</span> return -1;<a name="line.787"></a>
-<span class="sourceLineNo">788</span> }<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> // we found a registered nonce, but the procedure may not have been submitted yet.<a name="line.791"></a>
-<span class="sourceLineNo">792</span> // since the client expect the procedure to be submitted, spin here until it is.<a name="line.792"></a>
-<span class="sourceLineNo">793</span> final boolean traceEnabled = LOG.isTraceEnabled();<a name="line.793"></a>
-<span class="sourceLineNo">794</span> while (isRunning() &&<a name="line.794"></a>
-<span class="sourceLineNo">795</span> !(procedures.containsKey(oldProcId) || completed.containsKey(oldProcId)) &&<a name="line.795"></a>
-<span class="sourceLineNo">796</span> nonceKeysToProcIdsMap.containsKey(nonceKey)) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> if (traceEnabled) {<a name="line.797"></a>
-<span class="sourceLineNo">798</span> LOG.trace("Waiting for pid=" + oldProcId.longValue() + " to be submitted");<a name="line.798"></a>
-<span class="sourceLineNo">799</span> }<a name="line.799"></a>
-<span class="sourceLineNo">800</span> Threads.sleep(100);<a name="line.800"></a>
-<span class="sourceLineNo">801</span> }<a name="line.801"></a>
-<span class="sourceLineNo">802</span> return oldProcId.longValue();<a name="line.802"></a>
-<span class="sourceLineNo">803</span> }<a name="line.803"></a>
-<span class="sourceLineNo">804</span><a name="line.804"></a>
-<span class="sourceLineNo">805</span> /**<a name="line.805"></a>
-<span class="sourceLineNo">806</span> * Remove the NonceKey if the procedure was not submitted to the executor.<a name="line.806"></a>
-<span class="sourceLineNo">807</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> */<a name="line.808"></a>
-<span class="sourceLineNo">809</span> public void unregisterNonceIfProcedureWasNotSubmitted(final NonceKey nonceKey) {<a name="line.809"></a>
-<span class="sourceLineNo">810</span> if (nonceKey == null) {<a name="line.810"></a>
-<span class="sourceLineNo">811</span> return;<a name="line.811"></a>
-<span class="sourceLineNo">812</span> }<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> final Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.814"></a>
-<span class="sourceLineNo">815</span> if (procId == null) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span> return;<a name="line.816"></a>
-<span class="sourceLineNo">817</span> }<a name="line.817"></a>
-<span class="sourceLineNo">818</span><a name="line.818"></a>
-<span class="sourceLineNo">819</span> // if the procedure was not submitted, remove the nonce<a name="line.819"></a>
-<span class="sourceLineNo">820</span> if (!(procedures.containsKey(procId) || completed.containsKey(procId))) {<a name="line.820"></a>
-<span class="sourceLineNo">821</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.821"></a>
-<span class="sourceLineNo">822</span> }<a name="line.822"></a>
-<span class="sourceLineNo">823</span> }<a name="line.823"></a>
-<span class="sourceLineNo">824</span><a name="line.824"></a>
-<span class="sourceLineNo">825</span> /**<a name="line.825"></a>
-<span class="sourceLineNo">826</span> * If the failure failed before submitting it, we may want to give back the<a name="line.826"></a>
-<span class="sourceLineNo">827</span> * same error to the requests with the same nonceKey.<a name="line.827"></a>
-<span class="sourceLineNo">828</span> *<a name="line.828"></a>
-<span class="sourceLineNo">829</span> * @param nonceKey A unique identifier for this operation from the client or process<a name="line.829"></a>
-<span class="sourceLineNo">830</span> * @param procName name of the procedure, used to inform the user<a name="line.830"></a>
-<span class="sourceLineNo">831</span> * @param procOwner name of the owner of the procedure, used to inform the user<a name="line.831"></a>
-<span class="sourceLineNo">832</span> * @param exception the failure to report to the user<a name="line.832"></a>
-<span class="sourceLineNo">833</span> */<a name="line.833"></a>
-<span class="sourceLineNo">834</span> public void setFailureResultForNonce(NonceKey nonceKey, String procName, User procOwner,<a name="line.834"></a>
-<span class="sourceLineNo">835</span> IOException exception) {<a name="line.835"></a>
-<span class="sourceLineNo">836</span> if (nonceKey == null) {<a name="line.836"></a>
-<span class="sourceLineNo">837</span> return;<a name="line.837"></a>
-<span class="sourceLineNo">838</span> }<a name="line.838"></a>
-<span class="sourceLineNo">839</span><a name="line.839"></a>
-<span class="sourceLineNo">840</span> Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.840"></a>
-<span class="sourceLineNo">841</span> if (procId == null || completed.containsKey(procId)) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span> return;<a name="line.842"></a>
-<span class="sourceLineNo">843</span> }<a name="line.843"></a>
-<span class="sourceLineNo">844</span><a name="line.844"></a>
-<span class="sourceLineNo">845</span> completed.computeIfAbsent(procId, (key) -> {<a name="line.845"></a>
-<span class="sourceLineNo">846</span> Procedure<TEnvironment> proc = new FailedProcedure<>(procId.longValue(),<a name="line.846"></a>
-<span class="sourceLineNo">847</span> procName, procOwner, nonceKey, exception);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> return new CompletedProcedureRetainer<>(proc);<a name="line.849"></a>
-<span class="sourceLineNo">850</span> });<a name="line.850"></a>
-<span class="sourceLineNo">851</span> }<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> // ==========================================================================<a name="line.853"></a>
-<span class="sourceLineNo">854</span> // Submit/Abort Procedure<a name="line.854"></a>
-<span class="sourceLineNo">855</span> // ==========================================================================<a name="line.855"></a>
-<span class="sourceLineNo">856</span> /**<a name="line.856"></a>
-<span class="sourceLineNo">857</span> * Add a new root-procedure to the executor.<a name="line.857"></a>
-<span class="sourceLineNo">858</span> * @param proc the new procedure to execute.<a name="line.858"></a>
-<span class="sourceLineNo">859</span> * @return the procedure id, that can be used to monitor the operation<a name="line.859"></a>
-<span class="sourceLineNo">860</span> */<a name="line.860"></a>
-<span class="sourceLineNo">861</span> public long submitProcedure(Procedure<TEnvironment> proc) {<a name="line.861"></a>
-<span class="sourceLineNo">862</span> return submitProcedure(proc, null);<a name="line.862"></a>
-<span class="sourceLineNo">863</span> }<a name="line.863"></a>
-<span class="sourceLineNo">864</span><a name="line.864"></a>
-<span class="sourceLineNo">865</span> /**<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Bypass a procedure. If the procedure is set to bypass, all the logic in<a name="line.866"></a>
-<span class="sourceLineNo">867</span> * execute/rollback will be ignored and it will return success, whatever.<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * It is used to recover buggy stuck procedures, releasing the lock resources<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * and letting other procedures run. Bypassing one procedure (and its ancestors will<a name="line.869"></a>
-<span class="sourceLineNo">870</span> * be bypassed automatically) may leave the cluster in a middle state, e.g. region<a name="line.870"></a>
-<span class="sourceLineNo">871</span> * not assigned, or some hdfs files left behind. After getting rid of those stuck procedures,<a name="line.871"></a>
-<span class="sourceLineNo">872</span> * the operators may have to do some clean up on hdfs or schedule some assign procedures<a name="line.872"></a>
-<span class="sourceLineNo">873</span> * to let region online. DO AT YOUR OWN RISK.<a name="line.873"></a>
-<span class="sourceLineNo">874</span> * <p><a name="line.874"></a>
-<span class="sourceLineNo">875</span> * A procedure can be bypassed only if<a name="line.875"></a>
-<span class="sourceLineNo">876</span> * 1. The procedure is in state of RUNNABLE, WAITING, WAITING_TIMEOUT<a name="line.876"></a>
-<span class="sourceLineNo">877</span> * or it is a root procedure without any child.<a name="line.877"></a>
-<span class="sourceLineNo">878</span> * 2. No other worker thread is executing it<a name="line.878"></a>
-<span class="sourceLineNo">879</span> * 3. No child procedure has been submitted<a name="line.879"></a>
-<span class="sourceLineNo">880</span> *<a name="line.880"></a>
+<span class="sourceLineNo">020</span>import edu.umd.cs.findbugs.annotations.Nullable;<a name="line.20"></a>
+<span class="sourceLineNo">021</span>import java.io.IOException;<a name="line.21"></a>
+<span class="sourceLineNo">022</span>import java.io.UncheckedIOException;<a name="line.22"></a>
+<span class="sourceLineNo">023</span>import java.util.ArrayDeque;<a name="line.23"></a>
+<span class="sourceLineNo">024</span>import java.util.ArrayList;<a name="line.24"></a>
+<span class="sourceLineNo">025</span>import java.util.Arrays;<a name="line.25"></a>
+<span class="sourceLineNo">026</span>import java.util.Collection;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import java.util.Deque;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import java.util.HashSet;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import java.util.List;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import java.util.Set;<a name="line.30"></a>
+<span class="sourceLineNo">031</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.31"></a>
+<span class="sourceLineNo">032</span>import java.util.concurrent.CopyOnWriteArrayList;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>import java.util.concurrent.Executor;<a name="line.33"></a>
+<span class="sourceLineNo">034</span>import java.util.concurrent.Executors;<a name="line.34"></a>
+<span class="sourceLineNo">035</span>import java.util.concurrent.TimeUnit;<a name="line.35"></a>
+<span class="sourceLineNo">036</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.36"></a>
+<span class="sourceLineNo">037</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.37"></a>
+<span class="sourceLineNo">038</span>import java.util.concurrent.atomic.AtomicLong;<a name="line.38"></a>
+<span class="sourceLineNo">039</span>import java.util.stream.Collectors;<a name="line.39"></a>
+<span class="sourceLineNo">040</span>import java.util.stream.Stream;<a name="line.40"></a>
+<span class="sourceLineNo">041</span>import org.apache.hadoop.conf.Configuration;<a name="line.41"></a>
+<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.HConstants;<a name="line.42"></a>
+<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;<a name="line.43"></a>
+<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.log.HBaseMarkers;<a name="line.44"></a>
+<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.procedure2.Procedure.LockState;<a name="line.45"></a>
+<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;<a name="line.46"></a>
+<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;<a name="line.47"></a>
+<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureStoreListener;<a name="line.48"></a>
+<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.procedure2.util.StringUtils;<a name="line.49"></a>
+<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.security.User;<a name="line.50"></a>
+<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.51"></a>
+<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.util.IdLock;<a name="line.52"></a>
+<span class="sourceLineNo">053</span>import org.apache.hadoop.hbase.util.NonceKey;<a name="line.53"></a>
+<span class="sourceLineNo">054</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.54"></a>
+<span class="sourceLineNo">055</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.55"></a>
+<span class="sourceLineNo">056</span>import org.slf4j.Logger;<a name="line.56"></a>
+<span class="sourceLineNo">057</span>import org.slf4j.LoggerFactory;<a name="line.57"></a>
+<span class="sourceLineNo">058</span><a name="line.58"></a>
+<span class="sourceLineNo">059</span>import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;<a name="line.59"></a>
+<span class="sourceLineNo">060</span>import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;<a name="line.60"></a>
+<span class="sourceLineNo">061</span>import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;<a name="line.61"></a>
+<span class="sourceLineNo">062</span><a name="line.62"></a>
+<span class="sourceLineNo">063</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.63"></a>
+<span class="sourceLineNo">064</span><a name="line.64"></a>
+<span class="sourceLineNo">065</span>/**<a name="line.65"></a>
+<span class="sourceLineNo">066</span> * Thread Pool that executes the submitted procedures.<a name="line.66"></a>
+<span class="sourceLineNo">067</span> * The executor has a ProcedureStore associated.<a name="line.67"></a>
+<span class="sourceLineNo">068</span> * Each operation is logged and on restart the pending procedures are resumed.<a name="line.68"></a>
+<span class="sourceLineNo">069</span> *<a name="line.69"></a>
+<span class="sourceLineNo">070</span> * Unless the Procedure code throws an error (e.g. invalid user input)<a name="line.70"></a>
+<span class="sourceLineNo">071</span> * the procedure will complete (at some point in time), On restart the pending<a name="line.71"></a>
+<span class="sourceLineNo">072</span> * procedures are resumed and the once failed will be rolledback.<a name="line.72"></a>
+<span class="sourceLineNo">073</span> *<a name="line.73"></a>
+<span class="sourceLineNo">074</span> * The user can add procedures to the executor via submitProcedure(proc)<a name="line.74"></a>
+<span class="sourceLineNo">075</span> * check for the finished state via isFinished(procId)<a name="line.75"></a>
+<span class="sourceLineNo">076</span> * and get the result via getResult(procId)<a name="line.76"></a>
+<span class="sourceLineNo">077</span> */<a name="line.77"></a>
+<span class="sourceLineNo">078</span>@InterfaceAudience.Private<a name="line.78"></a>
+<span class="sourceLineNo">079</span>public class ProcedureExecutor<TEnvironment> {<a name="line.79"></a>
+<span class="sourceLineNo">080</span> private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);<a name="line.80"></a>
+<span class="sourceLineNo">081</span><a name="line.81"></a>
+<span class="sourceLineNo">082</span> public static final String CHECK_OWNER_SET_CONF_KEY = "hbase.procedure.check.owner.set";<a name="line.82"></a>
+<span class="sourceLineNo">083</span> private static final boolean DEFAULT_CHECK_OWNER_SET = false;<a name="line.83"></a>
+<span class="sourceLineNo">084</span><a name="line.84"></a>
+<span class="sourceLineNo">085</span> public static final String WORKER_KEEP_ALIVE_TIME_CONF_KEY =<a name="line.85"></a>
+<span class="sourceLineNo">086</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.86"></a>
+<span class="sourceLineNo">087</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.87"></a>
+<span class="sourceLineNo">088</span><a name="line.88"></a>
+<span class="sourceLineNo">089</span> public static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.89"></a>
+<span class="sourceLineNo">090</span> static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.90"></a>
+<span class="sourceLineNo">091</span><a name="line.91"></a>
+<span class="sourceLineNo">092</span> public static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.92"></a>
+<span class="sourceLineNo">093</span> static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.93"></a>
+<span class="sourceLineNo">094</span><a name="line.94"></a>
+<span class="sourceLineNo">095</span> /**<a name="line.95"></a>
+<span class="sourceLineNo">096</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.96"></a>
+<span class="sourceLineNo">097</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.97"></a>
+<span class="sourceLineNo">098</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.98"></a>
+<span class="sourceLineNo">099</span> */<a name="line.99"></a>
+<span class="sourceLineNo">100</span> volatile Testing testing = null;<a name="line.100"></a>
+<span class="sourceLineNo">101</span><a name="line.101"></a>
+<span class="sourceLineNo">102</span> /**<a name="line.102"></a>
+<span class="sourceLineNo">103</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.103"></a>
+<span class="sourceLineNo">104</span> */<a name="line.104"></a>
+<span class="sourceLineNo">105</span> public static class Testing {<a name="line.105"></a>
+<span class="sourceLineNo">106</span> protected volatile boolean killIfHasParent = true;<a name="line.106"></a>
+<span class="sourceLineNo">107</span> protected volatile boolean killIfSuspended = false;<a name="line.107"></a>
+<span class="sourceLineNo">108</span><a name="line.108"></a>
+<span class="sourceLineNo">109</span> /**<a name="line.109"></a>
+<span class="sourceLineNo">110</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.110"></a>
+<span class="sourceLineNo">111</span> * persisting all the state it needs to recover after a crash.<a name="line.111"></a>
+<span class="sourceLineNo">112</span> */<a name="line.112"></a>
+<span class="sourceLineNo">113</span> protected volatile boolean killBeforeStoreUpdate = false;<a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected volatile boolean toggleKillBeforeStoreUpdate = false;<a name="line.114"></a>
+<span class="sourceLineNo">115</span><a name="line.115"></a>
+<span class="sourceLineNo">116</span> /**<a name="line.116"></a>
+<span class="sourceLineNo">117</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.117"></a>
+<span class="sourceLineNo">118</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.118"></a>
+<span class="sourceLineNo">119</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.119"></a>
+<span class="sourceLineNo">120</span> */<a name="line.120"></a>
+<span class="sourceLineNo">121</span> protected volatile boolean killAfterStoreUpdate = false;<a name="line.121"></a>
+<span class="sourceLineNo">122</span> protected volatile boolean toggleKillAfterStoreUpdate = false;<a name="line.122"></a>
+<span class="sourceLineNo">123</span><a name="line.123"></a>
+<span class="sourceLineNo">124</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.124"></a>
+<span class="sourceLineNo">125</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.125"></a>
+<span class="sourceLineNo">126</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.126"></a>
+<span class="sourceLineNo">127</span> this.killBeforeStoreUpdate = !kill;<a name="line.127"></a>
+<span class="sourceLineNo">128</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span> return kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> }<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span> protected boolean shouldKillBeforeStoreUpdate(boolean isSuspended, boolean hasParent) {<a name="line.133"></a>
+<span class="sourceLineNo">134</span> if (isSuspended && !killIfSuspended) {<a name="line.134"></a>
+<span class="sourceLineNo">135</span> return false;<a name="line.135"></a>
+<span class="sourceLineNo">136</span> }<a name="line.136"></a>
+<span class="sourceLineNo">137</span> if (hasParent && !killIfHasParent) {<a name="line.137"></a>
+<span class="sourceLineNo">138</span> return false;<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span> return shouldKillBeforeStoreUpdate();<a name="line.140"></a>
+<span class="sourceLineNo">141</span> }<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.143"></a>
+<span class="sourceLineNo">144</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.144"></a>
+<span class="sourceLineNo">145</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.145"></a>
+<span class="sourceLineNo">146</span> this.killAfterStoreUpdate = !kill;<a name="line.146"></a>
+<span class="sourceLineNo">147</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.147"></a>
+<span class="sourceLineNo">148</span> }<a name="line.148"></a>
+<span class="sourceLineNo">149</span> return kill;<a name="line.149"></a>
+<span class="sourceLineNo">150</span> }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.152"></a>
+<span class="sourceLineNo">153</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span> }<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span> public interface ProcedureExecutorListener {<a name="line.157"></a>
+<span class="sourceLineNo">158</span> void procedureLoaded(long procId);<a name="line.158"></a>
+<span class="sourceLineNo">159</span> void procedureAdded(long procId);<a name="line.159"></a>
+<span class="sourceLineNo">160</span> void procedureFinished(long procId);<a name="line.160"></a>
+<span class="sourceLineNo">161</span> }<a name="line.161"></a>
+<span class="sourceLineNo">162</span><a name="line.162"></a>
+<span class="sourceLineNo">163</span> /**<a name="line.163"></a>
+<span class="sourceLineNo">164</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.164"></a>
+<span class="sourceLineNo">165</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.165"></a>
+<span class="sourceLineNo">166</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.166"></a>
+<span class="sourceLineNo">167</span> */<a name="line.167"></a>
+<span class="sourceLineNo">168</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.168"></a>
+<span class="sourceLineNo">169</span> new ConcurrentHashMap<>();<a name="line.169"></a>
+<span class="sourceLineNo">170</span><a name="line.170"></a>
+<span class="sourceLineNo">171</span> /**<a name="line.171"></a>
+<span class="sourceLineNo">172</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.172"></a>
+<span class="sourceLineNo">173</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.173"></a>
+<span class="sourceLineNo">174</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.174"></a>
+<span class="sourceLineNo">175</span> */<a name="line.175"></a>
+<span class="sourceLineNo">176</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.176"></a>
+<span class="sourceLineNo">177</span> new ConcurrentHashMap<>();<a name="line.177"></a>
+<span class="sourceLineNo">178</span><a name="line.178"></a>
+<span class="sourceLineNo">179</span> /**<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * Helper map to lookup the live procedures by ID.<a name="line.180"></a>
+<span class="sourceLineNo">181</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.181"></a>
+<span class="sourceLineNo">182</span> */<a name="line.182"></a>
+<span class="sourceLineNo">183</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.183"></a>
+<span class="sourceLineNo">184</span> new ConcurrentHashMap<>();<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> /**<a name="line.186"></a>
+<span class="sourceLineNo">187</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * contains every root procedure.<a name="line.188"></a>
+<span class="sourceLineNo">189</span> */<a name="line.189"></a>
+<span class="sourceLineNo">190</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.192"></a>
+<span class="sourceLineNo">193</span> new CopyOnWriteArrayList<>();<a name="line.193"></a>
+<span class="sourceLineNo">194</span><a name="line.194"></a>
+<span class="sourceLineNo">195</span> private Configuration conf;<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span> /**<a name="line.197"></a>
+<span class="sourceLineNo">198</span> * Created in the {@link #init(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.198"></a>
+<span class="sourceLineNo">199</span> * resource handling rather than observing in a #join is unexpected).<a name="line.199"></a>
+<span class="sourceLineNo">200</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.200"></a>
+<span class="sourceLineNo">201</span> * (Should be ok).<a name="line.201"></a>
+<span class="sourceLineNo">202</span> */<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private ThreadGroup threadGroup;<a name="line.203"></a>
+<span class="sourceLineNo">204</span><a name="line.204"></a>
+<span class="sourceLineNo">205</span> /**<a name="line.205"></a>
+<span class="sourceLineNo">206</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.206"></a>
+<span class="sourceLineNo">207</span> * resource handling rather than observing in a #join is unexpected).<a name="line.207"></a>
+<span class="sourceLineNo">208</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.208"></a>
+<span class="sourceLineNo">209</span> * (Should be ok).<a name="line.209"></a>
+<span class="sourceLineNo">210</span> */<a name="line.210"></a>
+<span class="sourceLineNo">211</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.211"></a>
+<span class="sourceLineNo">212</span><a name="line.212"></a>
+<span class="sourceLineNo">213</span> /**<a name="line.213"></a>
+<span class="sourceLineNo">214</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.214"></a>
+<span class="sourceLineNo">215</span> * resource handling rather than observing in a #join is unexpected).<a name="line.215"></a>
+<span class="sourceLineNo">216</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.216"></a>
+<span class="sourceLineNo">217</span> * (Should be ok).<a name="line.217"></a>
+<span class="sourceLineNo">218</span> */<a name="line.218"></a>
+<span class="sourceLineNo">219</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.219"></a>
+<span class="sourceLineNo">220</span><a name="line.220"></a>
+<span class="sourceLineNo">221</span> /**<a name="line.221"></a>
+<span class="sourceLineNo">222</span> * WorkerMonitor check for stuck workers and new worker thread when necessary, for example if<a name="line.222"></a>
+<span class="sourceLineNo">223</span> * there is no worker to assign meta, it will new worker thread for it, so it is very important.<a name="line.223"></a>
+<span class="sourceLineNo">224</span> * TimeoutExecutor execute many tasks like DeadServerMetricRegionChore RegionInTransitionChore<a name="line.224"></a>
+<span class="sourceLineNo">225</span> * and so on, some tasks may execute for a long time so will block other tasks like<a name="line.225"></a>
+<span class="sourceLineNo">226</span> * WorkerMonitor, so use a dedicated thread for executing WorkerMonitor.<a name="line.226"></a>
+<span class="sourceLineNo">227</span> */<a name="line.227"></a>
+<span class="sourceLineNo">228</span> private TimeoutExecutorThread<TEnvironment> workerMonitorExecutor;<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span> private int corePoolSize;<a name="line.230"></a>
+<span class="sourceLineNo">231</span> private int maxPoolSize;<a name="line.231"></a>
+<span class="sourceLineNo">232</span><a name="line.232"></a>
+<span class="sourceLineNo">233</span> private volatile long keepAliveTime;<a name="line.233"></a>
+<span class="sourceLineNo">234</span><a name="line.234"></a>
+<span class="sourceLineNo">235</span> /**<a name="line.235"></a>
+<span class="sourceLineNo">236</span> * Scheduler/Queue that contains runnable procedures.<a name="line.236"></a>
+<span class="sourceLineNo">237</span> */<a name="line.237"></a>
+<span class="sourceLineNo">238</span> private final ProcedureScheduler scheduler;<a name="line.238"></a>
+<span class="sourceLineNo">239</span><a name="line.239"></a>
+<span class="sourceLineNo">240</span> private final Executor forceUpdateExecutor = Executors.newSingleThreadExecutor(<a name="line.240"></a>
+<span class="sourceLineNo">241</span> new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.243"></a>
+<span class="sourceLineNo">244</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.244"></a>
+<span class="sourceLineNo">245</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.245"></a>
+<span class="sourceLineNo">246</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.246"></a>
+<span class="sourceLineNo">247</span> private final TEnvironment environment;<a name="line.247"></a>
+<span class="sourceLineNo">248</span> private final ProcedureStore store;<a name="line.248"></a>
+<span class="sourceLineNo">249</span><a name="line.249"></a>
+<span class="sourceLineNo">250</span> private final boolean checkOwnerSet;<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span> // To prevent concurrent execution of the same procedure.<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.253"></a>
+<span class="sourceLineNo">254</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.254"></a>
+<span class="sourceLineNo">255</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.255"></a>
+<span class="sourceLineNo">256</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.256"></a>
+<span class="sourceLineNo">257</span> // execution of the same procedure.<a name="line.257"></a>
+<span class="sourceLineNo">258</span> private final IdLock procExecutionLock = new IdLock();<a name="line.258"></a>
+<span class="sourceLineNo">259</span><a name="line.259"></a>
+<span class="sourceLineNo">260</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final ProcedureStore store) {<a name="line.261"></a>
+<span class="sourceLineNo">262</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span> private boolean isRootFinished(Procedure<?> proc) {<a name="line.265"></a>
+<span class="sourceLineNo">266</span> Procedure<?> rootProc = procedures.get(proc.getRootProcId());<a name="line.266"></a>
+<span class="sourceLineNo">267</span> return rootProc == null || rootProc.isFinished();<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span><a name="line.269"></a>
+<span class="sourceLineNo">270</span> private void forceUpdateProcedure(long procId) throws IOException {<a name="line.270"></a>
+<span class="sourceLineNo">271</span> IdLock.Entry lockEntry = procExecutionLock.getLockEntry(procId);<a name="line.271"></a>
+<span class="sourceLineNo">272</span> try {<a name="line.272"></a>
+<span class="sourceLineNo">273</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.273"></a>
+<span class="sourceLineNo">274</span> if (proc != null) {<a name="line.274"></a>
+<span class="sourceLineNo">275</span> if (proc.isFinished() && proc.hasParent() && isRootFinished(proc)) {<a name="line.275"></a>
+<span class="sourceLineNo">276</span> LOG.debug("Procedure {} has already been finished and parent is succeeded," +<a name="line.276"></a>
+<span class="sourceLineNo">277</span> " skip force updating", proc);<a name="line.277"></a>
+<span class="sourceLineNo">278</span> return;<a name="line.278"></a>
+<span class="sourceLineNo">279</span> }<a name="line.279"></a>
+<span class="sourceLineNo">280</span> } else {<a name="line.280"></a>
+<span class="sourceLineNo">281</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.281"></a>
+<span class="sourceLineNo">282</span> if (retainer == null || retainer.getProcedure() instanceof FailedProcedure) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span> LOG.debug("No pending procedure with id = {}, skip force updating.", procId);<a name="line.283"></a>
+<span class="sourceLineNo">284</span> return;<a name="line.284"></a>
+<span class="sourceLineNo">285</span> }<a name="line.285"></a>
+<span class="sourceLineNo">286</span> long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.286"></a>
+<span class="sourceLineNo">287</span> long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.287"></a>
+<span class="sourceLineNo">288</span> if (retainer.isExpired(System.currentTimeMillis(), evictTtl, evictAckTtl)) {<a name="line.288"></a>
+<span class="sourceLineNo">289</span> LOG.debug("Procedure {} has already been finished and expired, skip force updating",<a name="line.289"></a>
+<span class="sourceLineNo">290</span> procId);<a name="line.290"></a>
+<span class="sourceLineNo">291</span> return;<a name="line.291"></a>
+<span class="sourceLineNo">292</span> }<a name="line.292"></a>
+<span class="sourceLineNo">293</span> proc = retainer.getProcedure();<a name="line.293"></a>
+<span class="sourceLineNo">294</span> }<a name="line.294"></a>
+<span class="sourceLineNo">295</span> LOG.debug("Force update procedure {}", proc);<a name="line.295"></a>
+<span class="sourceLineNo">296</span> store.update(proc);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> } finally {<a name="line.297"></a>
+<span class="sourceLineNo">298</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.298"></a>
+<span class="sourceLineNo">299</span> }<a name="line.299"></a>
+<span class="sourceLineNo">300</span> }<a name="line.300"></a>
+<span class="sourceLineNo">301</span><a name="line.301"></a>
+<span class="sourceLineNo">302</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.302"></a>
+<span class="sourceLineNo">303</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.303"></a>
+<span class="sourceLineNo">304</span> this.environment = environment;<a name="line.304"></a>
+<span class="sourceLineNo">305</span> this.scheduler = scheduler;<a name="line.305"></a>
+<span class="sourceLineNo">306</span> this.store = store;<a name="line.306"></a>
+<span class="sourceLineNo">307</span> this.conf = conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.308"></a>
+<span class="sourceLineNo">309</span> refreshConfiguration(conf);<a name="line.309"></a>
+<span class="sourceLineNo">310</span> store.registerListener(new ProcedureStoreListener() {<a name="line.310"></a>
+<span class="sourceLineNo">311</span><a name="line.311"></a>
+<span class="sourceLineNo">312</span> @Override<a name="line.312"></a>
+<span class="sourceLineNo">313</span> public void forceUpdate(long[] procIds) {<a name="line.313"></a>
+<span class="sourceLineNo">314</span> Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> {<a name="line.314"></a>
+<span class="sourceLineNo">315</span> try {<a name="line.315"></a>
+<span class="sourceLineNo">316</span> forceUpdateProcedure(procId);<a name="line.316"></a>
+<span class="sourceLineNo">317</span> } catch (IOException e) {<a name="line.317"></a>
+<span class="sourceLineNo">318</span> LOG.warn("Failed to force update procedure with pid={}", procId);<a name="line.318"></a>
+<span class="sourceLineNo">319</span> }<a name="line.319"></a>
+<span class="sourceLineNo">320</span> }));<a name="line.320"></a>
+<span class="sourceLineNo">321</span> }<a name="line.321"></a>
+<span class="sourceLineNo">322</span> });<a name="line.322"></a>
+<span class="sourceLineNo">323</span> }<a name="line.323"></a>
+<span class="sourceLineNo">324</span><a name="line.324"></a>
+<span class="sourceLineNo">325</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.325"></a>
+<span class="sourceLineNo">326</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.326"></a>
+<span class="sourceLineNo">327</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.327"></a>
+<span class="sourceLineNo">328</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.328"></a>
+<span class="sourceLineNo">329</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.329"></a>
+<span class="sourceLineNo">330</span><a name="line.330"></a>
+<span class="sourceLineNo">331</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.331"></a>
+<span class="sourceLineNo">332</span> @Override<a name="line.332"></a>
+<span class="sourceLineNo">333</span> public void setMaxProcId(long maxProcId) {<a name="line.333"></a>
+<span class="sourceLineNo">334</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.334"></a>
+<span class="sourceLineNo">335</span> lastProcId.set(maxProcId);<a name="line.335"></a>
+<span class="sourceLineNo">336</span> }<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> @Override<a name="line.338"></a>
+<span class="sourceLineNo">339</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.339"></a>
+<span class="sourceLineNo">340</span> loadProcedures(procIter, abortOnCorruption);<a name="line.340"></a>
+<span class="sourceLineNo">341</span> }<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> @Override<a name="line.343"></a>
+<span class="sourceLineNo">344</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.344"></a>
+<span class="sourceLineNo">345</span> int corruptedCount = 0;<a name="line.345"></a>
+<span class="sourceLineNo">346</span> while (procIter.hasNext()) {<a name="line.346"></a>
+<span class="sourceLineNo">347</span> Procedure<?> proc = procIter.next();<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.error("Corrupt " + proc);<a name="line.348"></a>
+<span class="sourceLineNo">349</span> corruptedCount++;<a name="line.349"></a>
+<span class="sourceLineNo">350</span> }<a name="line.350"></a>
+<span class="sourceLineNo">351</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.352"></a>
+<span class="sourceLineNo">353</span> }<a name="line.353"></a>
+<span class="sourceLineNo">354</span> }<a name="line.354"></a>
+<span class="sourceLineNo">355</span> });<a name="line.355"></a>
+<span class="sourceLineNo">356</span> }<a name="line.356"></a>
+<span class="sourceLineNo">357</span><a name="line.357"></a>
+<span class="sourceLineNo">358</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> proc.restoreLock(getEnvironment());<a name="line.359"></a>
+<span class="sourceLineNo">360</span> restored.add(proc.getProcId());<a name="line.360"></a>
+<span class="sourceLineNo">361</span> }<a name="line.361"></a>
+<span class="sourceLineNo">362</span><a name="line.362"></a>
+<span class="sourceLineNo">363</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.363"></a>
+<span class="sourceLineNo">364</span> while (!stack.isEmpty()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span> restoreLock(stack.pop(), restored);<a name="line.365"></a>
+<span class="sourceLineNo">366</span> }<a name="line.366"></a>
+<span class="sourceLineNo">367</span> }<a name="line.367"></a>
+<span class="sourceLineNo">368</span><a name="line.368"></a>
+<span class="sourceLineNo">369</span> // Restore the locks for all the procedures.<a name="line.369"></a>
+<span class="sourceLineNo">370</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.370"></a>
+<span class="sourceLineNo">371</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.371"></a>
+<span class="sourceLineNo">372</span> // calling the acquireLock method for the parent procedure.<a name="line.372"></a>
+<span class="sourceLineNo">373</span> // The algorithm is straight-forward:<a name="line.373"></a>
+<span class="sourceLineNo">374</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.374"></a>
+<span class="sourceLineNo">375</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.375"></a>
+<span class="sourceLineNo">376</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.376"></a>
+<span class="sourceLineNo">377</span> // unless<a name="line.377"></a>
+<span class="sourceLineNo">378</span> // a. We have no parent, i.e, we are the root procedure<a name="line.378"></a>
+<span class="sourceLineNo">379</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.379"></a>
+<span class="sourceLineNo">380</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.380"></a>
+<span class="sourceLineNo">381</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.381"></a>
+<span class="sourceLineNo">382</span> private void restoreLocks() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> Set<Long> restored = new HashSet<>();<a name="line.383"></a>
+<span class="sourceLineNo">384</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.384"></a>
+<span class="sourceLineNo">385</span> procedures.values().forEach(proc -> {<a name="line.385"></a>
+<span class="sourceLineNo">386</span> for (;;) {<a name="line.386"></a>
+<span class="sourceLineNo">387</span> if (restored.contains(proc.getProcId())) {<a name="line.387"></a>
+<span class="sourceLineNo">388</span> restoreLocks(stack, restored);<a name="line.388"></a>
+<span class="sourceLineNo">389</span> return;<a name="line.389"></a>
+<span class="sourceLineNo">390</span> }<a name="line.390"></a>
+<span class="sourceLineNo">391</span> if (!proc.hasParent()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span> restoreLock(proc, restored);<a name="line.392"></a>
+<span class="sourceLineNo">393</span> restoreLocks(stack, restored);<a name="line.393"></a>
+<span class="sourceLineNo">394</span> return;<a name="line.394"></a>
+<span class="sourceLineNo">395</span> }<a name="line.395"></a>
+<span class="sourceLineNo">396</span> stack.push(proc);<a name="line.396"></a>
+<span class="sourceLineNo">397</span> proc = procedures.get(proc.getParentProcId());<a name="line.397"></a>
+<span class="sourceLineNo">398</span> }<a name="line.398"></a>
+<span class="sourceLineNo">399</span> });<a name="line.399"></a>
+<span class="sourceLineNo">400</span> }<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throws IOException {<a name="line.403"></a>
+<span class="sourceLineNo">404</span> // 1. Build the rollback stack<a name="line.404"></a>
+<span class="sourceLineNo">405</span> int runnableCount = 0;<a name="line.405"></a>
+<span class="sourceLineNo">406</span> int failedCount = 0;<a name="line.406"></a>
+<span class="sourceLineNo">407</span> int waitingCount = 0;<a name="line.407"></a>
+<span class="sourceLineNo">408</span> int waitingTimeoutCount = 0;<a name="line.408"></a>
+<span class="sourceLineNo">409</span> while (procIter.hasNext()) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> boolean finished = procIter.isNextFinished();<a name="line.410"></a>
+<span class="sourceLineNo">411</span> @SuppressWarnings("unchecked")<a name="line.411"></a>
+<span class="sourceLineNo">412</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.412"></a>
+<span class="sourceLineNo">413</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.413"></a>
+<span class="sourceLineNo">414</span> long procId = proc.getProcId();<a name="line.414"></a>
+<span class="sourceLineNo">415</span><a name="line.415"></a>
+<span class="sourceLineNo">416</span> if (finished) {<a name="line.416"></a>
+<span class="sourceLineNo">417</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.417"></a>
+<span class="sourceLineNo">418</span> LOG.debug("Completed {}", proc);<a name="line.418"></a>
+<span class="sourceLineNo">419</span> } else {<a name="line.419"></a>
+<span class="sourceLineNo">420</span> if (!proc.hasParent()) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.421"></a>
+<span class="sourceLineNo">422</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.422"></a>
+<span class="sourceLineNo">423</span> }<a name="line.423"></a>
+<span class="sourceLineNo">424</span><a name="line.424"></a>
+<span class="sourceLineNo">425</span> // add the procedure to the map<a name="line.425"></a>
+<span class="sourceLineNo">426</span> proc.beforeReplay(getEnvironment());<a name="line.426"></a>
+<span class="sourceLineNo">427</span> procedures.put(proc.getProcId(), proc);<a name="line.427"></a>
+<span class="sourceLineNo">428</span> switch (proc.getState()) {<a name="line.428"></a>
+<span class="sourceLineNo">429</span> case RUNNABLE:<a name="line.429"></a>
+<span class="sourceLineNo">430</span> runnableCount++;<a name="line.430"></a>
+<span class="sourceLineNo">431</span> break;<a name="line.431"></a>
+<span class="sourceLineNo">432</span> case FAILED:<a name="line.432"></a>
+<span class="sourceLineNo">433</span> failedCount++;<a name="line.433"></a>
+<span class="sourceLineNo">434</span> break;<a name="line.434"></a>
+<span class="sourceLineNo">435</span> case WAITING:<a name="line.435"></a>
+<span class="sourceLineNo">436</span> waitingCount++;<a name="line.436"></a>
+<span class="sourceLineNo">437</span> break;<a name="line.437"></a>
+<span class="sourceLineNo">438</span> case WAITING_TIMEOUT:<a name="line.438"></a>
+<span class="sourceLineNo">439</span> waitingTimeoutCount++;<a name="line.439"></a>
+<span class="sourceLineNo">440</span> break;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> default:<a name="line.441"></a>
+<span class="sourceLineNo">442</span> break;<a name="line.442"></a>
+<span class="sourceLineNo">443</span> }<a name="line.443"></a>
+<span class="sourceLineNo">444</span> }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span> if (nonceKey != null) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the map<a name="line.447"></a>
+<span class="sourceLineNo">448</span> }<a name="line.448"></a>
+<span class="sourceLineNo">449</span> }<a name="line.449"></a>
+<span class="sourceLineNo">450</span><a name="line.450"></a>
+<span class="sourceLineNo">451</span> // 2. Initialize the stacks: In the old implementation, for procedures in FAILED state, we will<a name="line.451"></a>
+<span class="sourceLineNo">452</span> // push it into the ProcedureScheduler directly to execute the rollback. But this does not work<a name="line.452"></a>
+<span class="sourceLineNo">453</span> // after we introduce the restore lock stage. For now, when we acquire a xlock, we will remove<a name="line.453"></a>
+<span class="sourceLineNo">454</span> // the queue from runQueue in scheduler, and then when a procedure which has lock access, for<a name="line.454"></a>
+<span class="sourceLineNo">455</span> // example, a sub procedure of the procedure which has the xlock, is pushed into the scheduler,<a name="line.455"></a>
+<span class="sourceLineNo">456</span> // we will add the queue back to let the workers poll from it. The assumption here is that, the<a name="line.456"></a>
+<span class="sourceLineNo">457</span> // procedure which has the xlock should have been polled out already, so when loading we can not<a name="line.457"></a>
+<span class="sourceLineNo">458</span> // add the procedure to scheduler first and then call acquireLock, since the procedure is still<a name="line.458"></a>
+<span class="sourceLineNo">459</span> // in the queue, and since we will remove the queue from runQueue, then no one can poll it out,<a name="line.459"></a>
+<span class="sourceLineNo">460</span> // then there is a dead lock<a name="line.460"></a>
+<span class="sourceLineNo">461</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnableCount);<a name="line.461"></a>
+<span class="sourceLineNo">462</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.462"></a>
+<span class="sourceLineNo">463</span> List<Procedure<TEnvironment>> waitingList = new ArrayList<>(waitingCount);<a name="line.463"></a>
+<span class="sourceLineNo">464</span> List<Procedure<TEnvironment>> waitingTimeoutList = new ArrayList<>(waitingTimeoutCount);<a name="line.464"></a>
+<span class="sourceLineNo">465</span> procIter.reset();<a name="line.465"></a>
+<span class="sourceLineNo">466</span> while (procIter.hasNext()) {<a name="line.466"></a>
+<span class="sourceLineNo">467</span> if (procIter.isNextFinished()) {<a name="line.467"></a>
+<span class="sourceLineNo">468</span> procIter.skipNext();<a name="line.468"></a>
+<span class="sourceLineNo">469</span> continue;<a name="line.469"></a>
+<span class="sourceLineNo">470</span> }<a name="line.470"></a>
+<span class="sourceLineNo">471</span><a name="line.471"></a>
+<span class="sourceLineNo">472</span> @SuppressWarnings("unchecked")<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.473"></a>
+<span class="sourceLineNo">474</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.474"></a>
+<span class="sourceLineNo">475</span> LOG.debug("Loading {}", proc);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Long rootProcId = getRootProcedureId(proc);<a name="line.476"></a>
+<span class="sourceLineNo">477</span> // The orphan procedures will be passed to handleCorrupted, so add an assert here<a name="line.477"></a>
+<span class="sourceLineNo">478</span> assert rootProcId != null;<a name="line.478"></a>
+<span class="sourceLineNo">479</span><a name="line.479"></a>
+<span class="sourceLineNo">480</span> if (proc.hasParent()) {<a name="line.480"></a>
+<span class="sourceLineNo">481</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.481"></a>
+<span class="sourceLineNo">482</span> if (parent != null && !proc.isFinished()) {<a name="line.482"></a>
+<span class="sourceLineNo">483</span> parent.incChildrenLatch();<a name="line.483"></a>
+<span class="sourceLineNo">484</span> }<a name="line.484"></a>
+<span class="sourceLineNo">485</span> }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> procStack.loadStack(proc);<a name="line.488"></a>
+<span class="sourceLineNo">489</span><a name="line.489"></a>
+<span class="sourceLineNo">490</span> proc.setRootProcId(rootProcId);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> switch (proc.getState()) {<a name="line.491"></a>
+<span class="sourceLineNo">492</span> case RUNNABLE:<a name="line.492"></a>
+<span class="sourceLineNo">493</span> runnableList.add(proc);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> break;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> case WAITING:<a name="line.495"></a>
+<span class="sourceLineNo">496</span> waitingList.add(proc);<a name="line.496"></a>
+<span class="sourceLineNo">497</span> break;<a name="line.497"></a>
+<span class="sourceLineNo">498</span> case WAITING_TIMEOUT:<a name="line.498"></a>
+<span class="sourceLineNo">499</span> waitingTimeoutList.add(proc);<a name="line.499"></a>
+<span class="sourceLineNo">500</span> break;<a name="line.500"></a>
+<span class="sourceLineNo">501</span> case FAILED:<a name="line.501"></a>
+<span class="sourceLineNo">502</span> failedList.add(proc);<a name="line.502"></a>
+<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
+<span class="sourceLineNo">504</span> case ROLLEDBACK:<a name="line.504"></a>
+<span class="sourceLineNo">505</span> case INITIALIZING:<a name="line.505"></a>
+<span class="sourceLineNo">506</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.506"></a>
+<span class="sourceLineNo">507</span> LOG.error(msg);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> throw new UnsupportedOperationException(msg);<a name="line.508"></a>
+<span class="sourceLineNo">509</span> default:<a name="line.509"></a>
+<span class="sourceLineNo">510</span> break;<a name="line.510"></a>
+<span class="sourceLineNo">511</span> }<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> // 3. Check the waiting procedures to see if some of them can be added to runnable.<a name="line.514"></a>
+<span class="sourceLineNo">515</span> waitingList.forEach(proc -> {<a name="line.515"></a>
+<span class="sourceLineNo">516</span> if (!proc.hasChildren()) {<a name="line.516"></a>
+<span class="sourceLineNo">517</span> // Normally, WAITING procedures should be waken by its children. But, there is a case that,<a name="line.517"></a>
+<span class="sourceLineNo">518</span> // all the children are successful and before they can wake up their parent procedure, the<a name="line.518"></a>
+<span class="sourceLineNo">519</span> // master was killed. So, during recovering the procedures from ProcedureWal, its children<a name="line.519"></a>
+<span class="sourceLineNo">520</span> // are not loaded because of their SUCCESS state. So we need to continue to run this WAITING<a name="line.520"></a>
+<span class="sourceLineNo">521</span> // procedure. But before executing, we need to set its state to RUNNABLE, otherwise, a<a name="line.521"></a>
+<span class="sourceLineNo">522</span> // exception will throw:<a name="line.522"></a>
+<span class="sourceLineNo">523</span> // Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.523"></a>
+<span class="sourceLineNo">524</span> // "NOT RUNNABLE! " + procedure.toString());<a name="line.524"></a>
+<span class="sourceLineNo">525</span> proc.setState(ProcedureState.RUNNABLE);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> runnableList.add(proc);<a name="line.526"></a>
+<span class="sourceLineNo">527</span> } else {<a name="line.527"></a>
+<span class="sourceLineNo">528</span> proc.afterReplay(getEnvironment());<a name="line.528"></a>
+<span class="sourceLineNo">529</span> }<a name="line.529"></a>
+<span class="sourceLineNo">530</span> });<a name="line.530"></a>
+<span class="sourceLineNo">531</span> // 4. restore locks<a name="line.531"></a>
+<span class="sourceLineNo">532</span> restoreLocks();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span> // 5. Push the procedures to the timeout executor<a name="line.534"></a>
+<span class="sourceLineNo">535</span> waitingTimeoutList.forEach(proc -> {<a name="line.535"></a>
+<span class="sourceLineNo">536</span> proc.afterReplay(getEnvironment());<a name="line.536"></a>
+<span class="sourceLineNo">537</span> timeoutExecutor.add(proc);<a name="line.537"></a>
+<span class="sourceLineNo">538</span> });<a name="line.538"></a>
+<span class="sourceLineNo">539</span><a name="line.539"></a>
+<span class="sourceLineNo">540</span> // 6. Push the procedure to the scheduler<a name="line.540"></a>
+<span class="sourceLineNo">541</span> failedList.forEach(scheduler::addBack);<a name="line.541"></a>
+<span class="sourceLineNo">542</span> runnableList.forEach(p -> {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> p.afterReplay(getEnvironment());<a name="line.543"></a>
+<span class="sourceLineNo">544</span> if (!p.hasParent()) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.545"></a>
+<span class="sourceLineNo">546</span> }<a name="line.546"></a>
+<span class="sourceLineNo">547</span> scheduler.addBack(p);<a name="line.547"></a>
+<span class="sourceLineNo">548</span> });<a name="line.548"></a>
+<span class="sourceLineNo">549</span> // After all procedures put into the queue, signal the worker threads.<a name="line.549"></a>
+<span class="sourceLineNo">550</span> // Otherwise, there is a race condition. See HBASE-21364.<a name="line.550"></a>
+<span class="sourceLineNo">551</span> scheduler.signalAll();<a name="line.551"></a>
+<span class="sourceLineNo">552</span> }<a name="line.552"></a>
+<span class="sourceLineNo">553</span><a name="line.553"></a>
+<span class="sourceLineNo">554</span> /**<a name="line.554"></a>
+<span class="sourceLineNo">555</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.555"></a>
+<span class="sourceLineNo">556</span> * <p/><a name="line.556"></a>
+<span class="sourceLineNo">557</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.557"></a>
+<span class="sourceLineNo">558</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.558"></a>
+<span class="sourceLineNo">559</span> * pending and in-progress procedures.<a name="line.559"></a>
+<span class="sourceLineNo">560</span> * @param numThreads number of threads available for procedure execution.<a name="line.560"></a>
+<span class="sourceLineNo">561</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.561"></a>
+<span class="sourceLineNo">562</span> * is found on replay. otherwise false.<a name="line.562"></a>
+<span class="sourceLineNo">563</span> */<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> // We have numThreads executor + one timer thread used for timing out<a name="line.565"></a>
+<span class="sourceLineNo">566</span> // procedures and triggering periodic procedures.<a name="line.566"></a>
+<span class="sourceLineNo">567</span> this.corePoolSize = numThreads;<a name="line.567"></a>
+<span class="sourceLineNo">568</span> this.maxPoolSize = 10 * numThreads;<a name="line.568"></a>
+<span class="sourceLineNo">569</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.569"></a>
+<span class="sourceLineNo">570</span> corePoolSize, maxPoolSize);<a name="line.570"></a>
+<span class="sourceLineNo">571</span><a name="line.571"></a>
+<span class="sourceLineNo">572</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.572"></a>
+<span class="sourceLineNo">573</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, "ProcExecTimeout");<a name="line.573"></a>
+<span class="sourceLineNo">574</span> this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, threadGroup, "WorkerMonitor");<a name="line.574"></a>
+<span class="sourceLineNo">575</span><a name="line.575"></a>
+<span class="sourceLineNo">576</span> // Create the workers<a name="line.576"></a>
+<span class="sourceLineNo">577</span> workerId.set(0);<a name="line.577"></a>
+<span class="sourceLineNo">578</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.578"></a>
+<span class="sourceLineNo">579</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span><a name="line.582"></a>
+<span class="sourceLineNo">583</span> long st, et;<a name="line.583"></a>
+<span class="sourceLineNo">584</span><a name="line.584"></a>
+<span class="sourceLineNo">585</span> // Acquire the store lease.<a name="line.585"></a>
+<span class="sourceLineNo">586</span> st = System.nanoTime();<a name="line.586"></a>
+<span class="sourceLineNo">587</span> store.recoverLease();<a name="line.587"></a>
+<span class="sourceLineNo">588</span> et = System.nanoTime();<a name="line.588"></a>
+<span class="sourceLineNo">589</span> LOG.info("Recovered {} lease in {}", store.getClass().getSimpleName(),<a name="line.589"></a>
+<span class="sourceLineNo">590</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.590"></a>
+<span class="sourceLineNo">591</span><a name="line.591"></a>
+<span class="sourceLineNo">592</span> // start the procedure scheduler<a name="line.592"></a>
+<span class="sourceLineNo">593</span> scheduler.start();<a name="line.593"></a>
+<span class="sourceLineNo">594</span><a name="line.594"></a>
+<span class="sourceLineNo">595</span> // TODO: Split in two steps.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> // TODO: Handle corrupted procedures (currently just a warn)<a name="line.596"></a>
+<span class="sourceLineNo">597</span> // The first one will make sure that we have the latest id,<a name="line.597"></a>
+<span class="sourceLineNo">598</span> // so we can start the threads and accept new procedures.<a name="line.598"></a>
+<span class="sourceLineNo">599</span> // The second step will do the actual load of old procedures.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> st = System.nanoTime();<a name="line.600"></a>
+<span class="sourceLineNo">601</span> load(abortOnCorruption);<a name="line.601"></a>
+<span class="sourceLineNo">602</span> et = System.nanoTime();<a name="line.602"></a>
+<span class="sourceLineNo">603</span> LOG.info("Loaded {} in {}", store.getClass().getSimpleName(),<a name="line.603"></a>
+<span class="sourceLineNo">604</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.604"></a>
+<span class="sourceLineNo">605</span> }<a name="line.605"></a>
+<span class="sourceLineNo">606</span><a name="line.606"></a>
+<span class="sourceLineNo">607</span> /**<a name="line.607"></a>
+<span class="sourceLineNo">608</span> * Start the workers.<a name="line.608"></a>
+<span class="sourceLineNo">609</span> */<a name="line.609"></a>
+<span class="sourceLineNo">610</span> public void startWorkers() throws IOException {<a name="line.610"></a>
+<span class="sourceLineNo">611</span> if (!running.compareAndSet(false, true)) {<a name="line.611"></a>
+<span class="sourceLineNo">612</span> LOG.warn("Already running");<a name="line.612"></a>
+<span class="sourceLineNo">613</span> return;<a name="line.613"></a>
+<span class="sourceLineNo">614</span> }<a name="line.614"></a>
+<span class="sourceLineNo">615</span> // Start the executors. Here we must have the lastProcId set.<a name="line.615"></a>
+<span class="sourceLineNo">616</span> LOG.trace("Start workers {}", workerThreads.size());<a name="line.616"></a>
+<span class="sourceLineNo">617</span> timeoutExecutor.start();<a name="line.617"></a>
+<span class="sourceLineNo">618</span> workerMonitorExecutor.start();<a name="line.618"></a>
+<span class="sourceLineNo">619</span> for (WorkerThread worker: workerThreads) {<a name="line.619"></a>
+<span class="sourceLineNo">620</span> worker.start();<a name="line.620"></a>
+<span class="sourceLineNo">621</span> }<a name="line.621"></a>
+<span class="sourceLineNo">622</span><a name="line.622"></a>
+<span class="sourceLineNo">623</span> // Internal chores<a name="line.623"></a>
+<span class="sourceLineNo">624</span> workerMonitorExecutor.add(new WorkerMonitor());<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> // Add completed cleaner chore<a name="line.626"></a>
+<span class="sourceLineNo">627</span> addChore(new CompletedProcedureCleaner<>(conf, store, procExecutionLock, completed,<a name="line.627"></a>
+<span class="sourceLineNo">628</span> nonceKeysToProcIdsMap));<a name="line.628"></a>
+<span class="sourceLineNo">629</span> }<a name="line.629"></a>
+<span class="sourceLineNo">630</span><a name="line.630"></a>
+<span class="sourceLineNo">631</span> public void stop() {<a name="line.631"></a>
+<span class="sourceLineNo">632</span> if (!running.getAndSet(false)) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span> return;<a name="line.633"></a>
+<span class="sourceLineNo">634</span> }<a name="line.634"></a>
+<span class="sourceLineNo">635</span><a name="line.635"></a>
+<span class="sourceLineNo">636</span> LOG.info("Stopping");<a name="line.636"></a>
+<span class="sourceLineNo">637</span> scheduler.stop();<a name="line.637"></a>
+<span class="sourceLineNo">638</span> timeoutExecutor.sendStopSignal();<a name="line.638"></a>
+<span class="sourceLineNo">639</span> workerMonitorExecutor.sendStopSignal();<a name="line.639"></a>
+<span class="sourceLineNo">640</span> }<a name="line.640"></a>
+<span class="sourceLineNo">641</span><a name="line.641"></a>
+<span class="sourceLineNo">642</span> @VisibleForTesting<a name="line.642"></a>
+<span class="sourceLineNo">643</span> public void join() {<a name="line.643"></a>
+<span class="sourceLineNo">644</span> assert !isRunning() : "expected not running";<a name="line.644"></a>
+<span class="sourceLineNo">645</span><a name="line.645"></a>
+<span class="sourceLineNo">646</span> // stop the timeout executor<a name="line.646"></a>
+<span class="sourceLineNo">647</span> timeoutExecutor.awaitTermination();<a name="line.647"></a>
+<span class="sourceLineNo">648</span> // stop the work monitor executor<a name="line.648"></a>
+<span class="sourceLineNo">649</span> workerMonitorExecutor.awaitTermination();<a name="line.649"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span> // stop the worker threads<a name="line.651"></a>
+<span class="sourceLineNo">652</span> for (WorkerThread worker: workerThreads) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span> worker.awaitTermination();<a name="line.653"></a>
+<span class="sourceLineNo">654</span> }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span> // Destroy the Thread Group for the executors<a name="line.656"></a>
+<span class="sourceLineNo">657</span> // TODO: Fix. #join is not place to destroy resources.<a name="line.657"></a>
+<span class="sourceLineNo">658</span> try {<a name="line.658"></a>
+<span class="sourceLineNo">659</span> threadGroup.destroy();<a name="line.659"></a>
+<span class="sourceLineNo">660</span> } catch (IllegalThreadStateException e) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span> LOG.error("ThreadGroup {} contains running threads; {}: See STDOUT",<a name="line.661"></a>
+<span class="sourceLineNo">662</span> this.threadGroup, e.getMessage());<a name="line.662"></a>
+<span class="sourceLineNo">663</span> // This dumps list of threads on STDOUT.<a name="line.663"></a>
+<span class="sourceLineNo">664</span> this.threadGroup.list();<a name="line.664"></a>
+<span class="sourceLineNo">665</span> }<a name="line.665"></a>
+<span class="sourceLineNo">666</span><a name="line.666"></a>
+<span class="sourceLineNo">667</span> // reset the in-memory state for testing<a name="line.667"></a>
+<span class="sourceLineNo">668</span> completed.clear();<a name="line.668"></a>
+<span class="sourceLineNo">669</span> rollbackStack.clear();<a name="line.669"></a>
+<span class="sourceLineNo">670</span> procedures.clear();<a name="line.670"></a>
+<span class="sourceLineNo">671</span> nonceKeysToProcIdsMap.clear();<a name="line.671"></a>
+<span class="sourceLineNo">672</span> scheduler.clear();<a name="line.672"></a>
+<span class="sourceLineNo">673</span> lastProcId.set(-1);<a name="line.673"></a>
+<span class="sourceLineNo">674</span> }<a name="line.674"></a>
+<span class="sourceLineNo">675</span><a name="line.675"></a>
+<span class="sourceLineNo">676</span> public void refreshConfiguration(final Configuration conf) {<a name="line.676"></a>
+<span class="sourceLineNo">677</span> this.conf = conf;<a name="line.677"></a>
+<span class="sourceLineNo">678</span> setKeepAliveTime(conf.getLong(WORKER_KEEP_ALIVE_TIME_CONF_KEY,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> DEFAULT_WORKER_KEEP_ALIVE_TIME), TimeUnit.MILLISECONDS);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> }<a name="line.680"></a>
+<span class="sourceLineNo">681</span><a name="line.681"></a>
+<span class="sourceLineNo">682</span> // ==========================================================================<a name="line.682"></a>
+<span class="sourceLineNo">683</span> // Accessors<a name="line.683"></a>
+<span class="sourceLineNo">684</span> // ==========================================================================<a name="line.684"></a>
+<span class="sourceLineNo">685</span> public boolean isRunning() {<a name="line.685"></a>
+<span class="sourceLineNo">686</span> return running.get();<a name="line.686"></a>
+<span class="sourceLineNo">687</span> }<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span> /**<a name="line.689"></a>
+<span class="sourceLineNo">690</span> * @return the current number of worker threads.<a name="line.690"></a>
+<span class="sourceLineNo">691</span> */<a name="line.691"></a>
+<span class="sourceLineNo">692</span> public int getWorkerThreadCount() {<a name="line.692"></a>
+<span class="sourceLineNo">693</span> return workerThreads.size();<a name="line.693"></a>
+<span class="sourceLineNo">694</span> }<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span> /**<a name="line.696"></a>
+<span class="sourceLineNo">697</span> * @return the core pool size settings.<a name="line.697"></a>
+<span class="sourceLineNo">698</span> */<a name="line.698"></a>
+<span class="sourceLineNo">699</span> public int getCorePoolSize() {<a name="line.699"></a>
+<span class="sourceLineNo">700</span> return corePoolSize;<a name="line.700"></a>
+<span class="sourceLineNo">701</span> }<a name="line.701"></a>
+<span class="sourceLineNo">702</span><a name="line.702"></a>
+<span class="sourceLineNo">703</span> public int getActiveExecutorCount() {<a name="line.703"></a>
+<span class="sourceLineNo">704</span> return activeExecutorCount.get();<a name="line.704"></a>
+<span class="sourceLineNo">705</span> }<a name="line.705"></a>
+<span class="sourceLineNo">706</span><a name="line.706"></a>
+<span class="sourceLineNo">707</span> public TEnvironment getEnvironment() {<a name="line.707"></a>
+<span class="sourceLineNo">708</span> return this.environment;<a name="line.708"></a>
+<span class="sourceLineNo">709</span> }<a name="line.709"></a>
+<span class="sourceLineNo">710</span><a name="line.710"></a>
+<span class="sourceLineNo">711</span> public ProcedureStore getStore() {<a name="line.711"></a>
+<span class="sourceLineNo">712</span> return this.store;<a name="line.712"></a>
+<span class="sourceLineNo">713</span> }<a name="line.713"></a>
+<span class="sourceLineNo">714</span><a name="line.714"></a>
+<span class="sourceLineNo">715</span> ProcedureScheduler getScheduler() {<a name="line.715"></a>
+<span class="sourceLineNo">716</span> return scheduler;<a name="line.716"></a>
+<span class="sourceLineNo">717</span> }<a name="line.717"></a>
+<span class="sourceLineNo">718</span><a name="line.718"></a>
+<span class="sourceLineNo">719</span> public void setKeepAliveTime(final long keepAliveTime, final TimeUnit timeUnit) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span> this.keepAliveTime = timeUnit.toMillis(keepAliveTime);<a name="line.720"></a>
+<span class="sourceLineNo">721</span> this.scheduler.signalAll();<a name="line.721"></a>
+<span class="sourceLineNo">722</span> }<a name="line.722"></a>
+<span class="sourceLineNo">723</span><a name="line.723"></a>
+<span class="sourceLineNo">724</span> public long getKeepAliveTime(final TimeUnit timeUnit) {<a name="line.724"></a>
+<span class="sourceLineNo">725</span> return timeUnit.convert(keepAliveTime, TimeUnit.MILLISECONDS);<a name="line.725"></a>
+<span class="sourceLineNo">726</span> }<a name="line.726"></a>
+<span class="sourceLineNo">727</span><a name="line.727"></a>
+<span class="sourceLineNo">728</span> // ==========================================================================<a name="line.728"></a>
+<span class="sourceLineNo">729</span> // Submit/Remove Chores<a name="line.729"></a>
+<span class="sourceLineNo">730</span> // ==========================================================================<a name="line.730"></a>
+<span class="sourceLineNo">731</span><a name="line.731"></a>
+<span class="sourceLineNo">732</span> /**<a name="line.732"></a>
+<span class="sourceLineNo">733</span> * Add a chore procedure to the executor<a name="line.733"></a>
+<span class="sourceLineNo">734</span> * @param chore the chore to add<a name="line.734"></a>
+<span class="sourceLineNo">735</span> */<a name="line.735"></a>
+<span class="sourceLineNo">736</span> public void addChore(@Nullable ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.736"></a>
+<span class="sourceLineNo">737</span> if (chore == null) {<a name="line.737"></a>
+<span class="sourceLineNo">738</span> return;<a name="line.738"></a>
+<span class="sourceLineNo">739</span> }<a name="line.739"></a>
+<span class="sourceLineNo">740</span> chore.setState(ProcedureState.WAITING_TIMEOUT);<a name="line.740"></a>
+<span class="sourceLineNo">741</span> timeoutExecutor.add(chore);<a name="line.741"></a>
+<span class="sourceLineNo">742</span> }<a name="line.742"></a>
+<span class="sourceLineNo">743</span><a name="line.743"></a>
+<span class="sourceLineNo">744</span> /**<a name="line.744"></a>
+<span class="sourceLineNo">745</span> * Remove a chore procedure from the executor<a name="line.745"></a>
+<span class="sourceLineNo">746</span> * @param chore the chore to remove<a name="line.746"></a>
+<span class="sourceLineNo">747</span> * @return whether the chore is removed, or it will be removed later<a name="line.747"></a>
+<span class="sourceLineNo">748</span> */<a name="line.748"></a>
+<span class="sourceLineNo">749</span> public boolean removeChore(@Nullable ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.749"></a>
+<span class="sourceLineNo">750</span> if (chore == null) {<a name="line.750"></a>
+<span class="sourceLineNo">751</span> return true;<a name="line.751"></a>
+<span class="sourceLineNo">752</span> }<a name="line.752"></a>
+<span class="sourceLineNo">753</span> chore.setState(ProcedureState.SUCCESS);<a name="line.753"></a>
+<span class="sourceLineNo">754</span> return timeoutExecutor.remove(chore);<a name="line.754"></a>
+<span class="sourceLineNo">755</span> }<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span> // ==========================================================================<a name="line.757"></a>
+<span class="sourceLineNo">758</span> // Nonce Procedure helpers<a name="line.758"></a>
+<span class="sourceLineNo">759</span> // ==========================================================================<a name="line.759"></a>
+<span class="sourceLineNo">760</span> /**<a name="line.760"></a>
+<span class="sourceLineNo">761</span> * Create a NonceKey from the specified nonceGroup and nonce.<a name="line.761"></a>
+<span class="sourceLineNo">762</span> * @param nonceGroup the group to use for the {@link NonceKey}<a name="line.762"></a>
+<span class="sourceLineNo">763</span> * @param nonce the nonce to use in the {@link NonceKey}<a name="line.763"></a>
+<span class="sourceLineNo">764</span> * @return the generated NonceKey<a name="line.764"></a>
+<span class="sourceLineNo">765</span> */<a name="line.765"></a>
+<span class="sourceLineNo">766</span> public NonceKey createNonceKey(final long nonceGroup, final long nonce) {<a name="line.766"></a>
+<span class="sourceLineNo">767</span> return (nonce == HConstants.NO_NONCE) ? null : new NonceKey(nonceGroup, nonce);<a name="line.767"></a>
+<span class="sourceLineNo">768</span> }<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span> /**<a name="line.770"></a>
+<span class="sourceLineNo">771</span> * Register a nonce for a procedure that is going to be submitted.<a name="line.771"></a>
+<span class="sourceLineNo">772</span> * A procId will be reserved and on submitProcedure(),<a name="line.772"></a>
+<span class="sourceLineNo">773</span> * the procedure with the specified nonce will take the reserved ProcId.<a name="line.773"></a>
+<span class="sourceLineNo">774</span> * If someone already reserved the nonce, this method will return the procId reserved,<a name="line.774"></a>
+<span class="sourceLineNo">775</span> * otherwise an invalid procId will be returned. and the caller should procede<a name="line.775"></a>
+<span class="sourceLineNo">776</span> * and submit the procedure.<a name="line.776"></a>
+<span class="sourceLineNo">777</span> *<a name="line.777"></a>
+<span class="sourceLineNo">778</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.778"></a>
+<span class="sourceLineNo">779</span> * @return the procId associated with the nonce, if any otherwise an invalid procId.<a name="line.779"></a>
+<span class="sourceLineNo">780</span> */<a name="line.780"></a>
+<span class="sourceLineNo">781</span> public long registerNonce(final NonceKey nonceKey) {<a name="line.781"></a>
+<span class="sourceLineNo">782</span> if (nonceKey == null) {<a name="line.782"></a>
+<span class="sourceLineNo">783</span> return -1;<a name="line.783"></a>
+<span class="sourceLineNo">784</span> }<a name="line.784"></a>
+<span class="sourceLineNo">785</span><a name="line.785"></a>
+<span class="sourceLineNo">786</span> // check if we have already a Reserved ID for the nonce<a name="line.786"></a>
+<span class="sourceLineNo">787</span> Long oldProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.787"></a>
+<span class="sourceLineNo">788</span> if (oldProcId == null) {<a name="line.788"></a>
+<span class="sourceLineNo">789</span> // reserve a new Procedure ID, this will be associated with the nonce<a name="line.789"></a>
+<span class="sourceLineNo">790</span> // and the procedure submitted with the specified nonce will use this ID.<a name="line.790"></a>
+<span class="sourceLineNo">791</span> final long newProcId = nextProcId();<a name="line.791"></a>
+<span class="sourceLineNo">792</span> oldProcId = nonceKeysToProcIdsMap.putIfAbsent(nonceKey, newProcId);<a name="line.792"></a>
+<span class="sourceLineNo">793</span> if (oldProcId == null) {<a name="line.793"></a>
+<span class="sourceLineNo">794</span> return -1;<a name="line.794"></a>
+<span class="sourceLineNo">795</span> }<a name="line.795"></a>
+<span class="sourceLineNo">796</span> }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span> // we found a registered nonce, but the procedure may not have been submitted yet.<a name="line.798"></a>
+<span class="sourceLineNo">799</span> // since the client expect the procedure to be submitted, spin here until it is.<a name="line.799"></a>
+<span class="sourceLineNo">800</span> final boolean traceEnabled = LOG.isTraceEnabled();<a name="line.800"></a>
+<span class="sourceLineNo">801</span> while (isRunning() &&<a name="line.801"></a>
+<span class="sourceLineNo">802</span> !(procedures.containsKey(oldProcId) || completed.containsKey(oldProcId)) &&<a name="line.802"></a>
+<span class="sourceLineNo">803</span> nonceKeysToProcIdsMap.containsKey(nonceKey)) {<a name="line.803"></a>
+<span class="sourceLineNo">804</span> if (traceEnabled) {<a name="line.804"></a>
+<span class="sourceLineNo">805</span> LOG.trace("Waiting for pid=" + oldProcId.longValue() + " to be submitted");<a name="line.805"></a>
+<span class="sourceLineNo">806</span> }<a name="line.806"></a>
+<span class="sourceLineNo">807</span> Threads.sleep(100);<a name="line.807"></a>
+<span class="sourceLineNo">808</span> }<a name="line.808"></a>
+<span class="sourceLineNo">809</span> return oldProcId.longValue();<a name="line.809"></a>
+<span class="sourceLineNo">810</span> }<a name="line.810"></a>
+<span class="sourceLineNo">811</span><a name="line.811"></a>
+<span class="sourceLineNo">812</span> /**<a name="line.812"></a>
+<span class="sourceLineNo">813</span> * Remove the NonceKey if the procedure was not submitted to the executor.<a name="line.813"></a>
+<span class="sourceLineNo">814</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.814"></a>
+<span class="sourceLineNo">815</span> */<a name="line.815"></a>
+<span class="sourceLineNo">816</span> public void unregisterNonceIfProcedureWasNotSubmitted(final NonceKey nonceKey) {<a name="line.816"></a>
+<span class="sourceLineNo">817</span> if (nonceKey == null) {<a name="line.817"></a>
+<span class="sourceLineNo">818</span> return;<a name="line.818"></a>
+<span class="sourceLineNo">819</span> }<a name="line.819"></a>
+<span class="sourceLineNo">820</span><a name="line.820"></a>
+<span class="sourceLineNo">821</span> final Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.821"></a>
+<span class="sourceLineNo">822</span> if (procId == null) {<a name="line.822"></a>
+<span class="sourceLineNo">823</span> return;<a name="line.823"></a>
+<span class="sourceLineNo">824</span> }<a name="line.824"></a>
+<span class="sourceLineNo">825</span><a name="line.825"></a>
+<span class="sourceLineNo">826</span> // if the procedure was not submitted, remove the nonce<a name="line.826"></a>
+<span class="sourceLineNo">827</span> if (!(procedures.containsKey(procId) || completed.containsKey(procId))) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.828"></a>
+<span class="sourceLineNo">829</span> }<a name="line.829"></a>
+<span class="sourceLineNo">830</span> }<a name="line.830"></a>
+<span class="sourceLineNo">831</span><a name="line.831"></a>
+<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
+<span class="sourceLineNo">833</span> * If the failure failed before submitting it, we may want to give back the<a name="line.833"></a>
+<span class="sourceLineNo">834</span> * same error to the requests with the same nonceKey.<a name="line.834"></a>
+<span class="sourceLineNo">835</span> *<a name="line.835"></a>
+<span class="sourceLineNo">836</span> * @param nonceKey A unique identifier for this operation from the client or process<a name="line.836"></a>
+<span class="sourceLineNo">837</span> * @param procName name of the procedure, used to inform the user<a name="line.837"></a>
+<span class="sourceLineNo">838</span> * @param procOwner name of the owner of the procedure, used to inform the user<a name="line.838"></a>
+<span class="sourceLineNo">839</span> * @param exception the failure to report to the user<a name="line.839"></a>
+<span class="sourceLineNo">840</span> */<a name="line.840"></a>
+<span class="sourceLineNo">841</span> public void setFailureResultForNonce(NonceKey nonceKey, String procName, User procOwner,<a name="line.841"></a>
+<span class="sourceLineNo">842</span> IOException exception) {<a name="line.842"></a>
+<span class="sourceLineNo">843</span> if (nonceKey == null) {<a name="line.843"></a>
+<span class="sourceLineNo">844</span> return;<a name="line.844"></a>
+<span class="sourceLineNo">845</span> }<a name="line.845"></a>
+<span class="sourceLineNo">846</span><a name="line.846"></a>
+<span class="sourceLineNo">847</span> Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.847"></a>
+<span class="sourceLineNo">848</span> if (procId == null || completed.containsKey(procId)) {<a name="line.848"></a>
+<span class="sourceLineNo">849</span> return;<a name="line.849"></a>
+<span class="sourceLineNo">850</span> }<a name="line.850"></a>
+<span class="sourceLineNo">851</span><a name="line.851"></a>
+<span class="sourceLineNo">852</span> completed.computeIfAbsent(procId, (key) -> {<a name="line.852"></a>
+<span class="sourceLineNo">853</span> Procedure<TEnvironment> proc = new FailedProcedure<>(procId.longValue(),<a name="line.853"></a>
+<span class="sourceLineNo">854</span> procName, procOwner, nonceKey, exception);<a name="line.854"></a>
+<span class="sourceLineNo">855</span><a name="line.855"></a>
+<span class="sourceLineNo">856</span> return new CompletedProcedureRetainer<>(proc);<a name="line.856"></a>
+<span class="sourceLineNo">857</span> });<a name="line.857"></a>
+<span class="sourceLineNo">858</span> }<a name="line.858"></a>
+<span class="sourceLineNo">859</span><a name="line.859"></a>
+<span class="sourceLineNo">860</span> // ==========================================================================<a name="line.860"></a>
+<span class="sourceLineNo">861</span> // Submit/Abort Procedure<a name="line.861"></a>
+<span class="sourceLineNo">862</span> // ==========================================================================<a name="line.862"></a>
+<span class="sourceLineNo">863</span> /**<a name="line.863"></a>
+<span class="sourceLineNo">864</span> * Add a new root-procedure to the executor.<a name="line.864"></a>
+<span class="sourceLineNo">865</span> * @param proc the new procedure to execute.<a name="line.865"></a>
+<span class="sourceLineNo">866</span> * @return the procedure id, that can be used to monitor the operation<a name="line.866"></a>
+<span class="sourceLineNo">867</span> */<a name="line.867"></a>
+<span class="sourceLineNo">868</span> public long submitProcedure(Procedure<TEnvironment> proc) {<a name="line.868"></a>
+<span class="sourceLineNo">869</span> return submitProcedure(proc, null);<a name="line.869"></a>
+<span class="sourceLineNo">870</span> }<a name="line.870"></a>
+<span class="sourceLineNo">871</span><a name="line.871"></a>
+<span class="sourceLineNo">872</span> /**<a name="line.872"></a>
+<span class="sourceLineNo">873</span> * Bypass a procedure. If the procedure is set to bypass, all the logic in<a name="line.873"></a>
+<span class="sourceLineNo">874</span> * execute/rollback will be ignored and it will return success, whatever.<a name="line.874"></a>
+<span class="sourceLineNo">875</span> * It is used to recover buggy stuck procedures, releasing the lock resources<a name="line.875"></a>
+<span class="sourceLineNo">876</span> * and letting other procedures run. Bypassing one procedure (and its ancestors will<a name="line.876"></a>
+<span class="sourceLineNo">877</span> * be bypassed automatically) may leave the cluster in a middle state, e.g. region<a name="line.877"></a>
+<span class="sourceLineNo">878</span> * not assigned, or some hdfs files left behind. After getting rid of those stuck procedures,<a name="line.878"></a>
+<span class="sourceLineNo">879</span> * the operators may have to do some clean up on hdfs or schedule some assign procedures<a name="line.879"></a>
+<span class="sourceLineNo">880</span> * to let region online. DO AT YOUR OWN RISK.<a name="line.880"></a>
<span class="sourceLineNo">881</span> * <p><a name="line.881"></a>
-<span class="sourceLineNo">882</span> * If all the requirements are meet, the procedure and its ancestors will be<a name="line.882"></a>
-<span class="sourceLineNo">883</span> * bypassed and persisted to WAL.<a name="line.883"></a>
-<span class="sourceLineNo">884</span> *<a name="line.884"></a>
-<span class="sourceLineNo">885</span> * <p><a name="line.885"></a>
-<span class="sourceLineNo">886</span> * If the procedure is in WAITING state, will set it to RUNNABLE add it to run queue.<a name="line.886"></a>
-<span class="sourceLineNo">887</span> * TODO: What about WAITING_TIMEOUT?<a name="line.887"></a>
-<span class="sourceLineNo">888</span> * @param pids the procedure id<a name="line.888"></a>
-<span class="sourceLineNo">889</span> * @param lockWait time to wait lock<a name="line.889"></a>
-<span class="sourceLineNo">890</span> * @param force if force set to true, we will bypass the procedure even if it is executing.<a name="line.890"></a>
-<span class="sourceLineNo">891</span> * This is for procedures which can't break out during executing(due to bug, mostly)<a name="line.891"></a>
-<span class="sourceLineNo">892</span> * In this case, bypassing the procedure is not enough, since it is already stuck<a name="line.892"></a>
-<span class="sourceLineNo">893</span> * there. We need to restart the master after bypassing, and letting the problematic<a name="line.893"></a>
-<span class="sourceLineNo">894</span> * procedure to execute wth bypass=true, so in that condition, the procedure can be<a name="line.894"></a>
-<span class="sourceLineNo">895</span> * successfully bypassed.<a name="line.895"></a>
-<span class="sourceLineNo">896</span> * @param recursive We will do an expensive search for children of each pid. EXPENSIVE!<a name="line.896"></a>
-<span class="sourceLineNo">897</span> * @return true if bypass success<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @throws IOException IOException<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> public List<Boolean> bypassProcedure(List<Long> pids, long lockWait, boolean force,<a name="line.900"></a>
-<span class="sourceLineNo">901</span> boolean recursive)<a name="line.901"></a>
-<span class="sourceLineNo">902</span> throws IOException {<a name="line.902"></a>
-<span class="sourceLineNo">903</span> List<Boolean> result = new ArrayList<Boolean>(pids.size());<a name="line.903"></a>
-<span class="sourceLineNo">904</span> for(long pid: pids) {<a name="line.904"></a>
-<span class="sourceLineNo">905</span> result.add(bypassProcedure(pid, lockWait, force, recursive));<a name="line.905"></a>
-<span class="sourceLineNo">906</span> }<a name="line.906"></a>
-<span class="sourceLineNo">907</span> return result;<a name="line.907"></a>
-<span class="sourceLineNo">908</span> }<a name="line.908"></a>
-<span class="sourceLineNo">909</span><a name="line.909"></a>
-<span class="sourceLineNo">910</span> boolean bypassProcedure(long pid, long lockWait, boolean override, boolean recursive)<a name="line.910"></a>
-<span class="sourceLineNo">911</span> throws IOException {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> Preconditions.checkArgument(lockWait > 0, "lockWait should be positive");<a name="line.912"></a>
-<span class="sourceLineNo">913</span> final Procedure<TEnvironment> procedure = getProcedure(pid);<a name="line.913"></a>
-<span class="sourceLineNo">914</span> if (procedure == null) {<a name="line.914"></a>
-<span class="sourceLineNo">915</span> LOG.debug("Procedure pid={} does not exist, skipping bypass", pid);<a name="line.915"></a>
-<span class="sourceLineNo">916</span> return false;<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span><a name="line.918"></a>
-<span class="sourceLineNo">919</span> LOG.debug("Begin bypass {} with lockWait={}, override={}, recursive={}",<a name="line.919"></a>
-<span class="sourceLineNo">920</span> procedure, lockWait, override, recursive);<a name="line.920"></a>
-<span class="sourceLineNo">921</span> IdLock.Entry lockEntry = procExecutionLock.tryLockEntry(procedure.getProcId(), lockWait);<a name="line.921"></a>
-<span class="sourceLineNo">922</span> if (lockEntry == null && !override) {<a name="line.922"></a>
-<span class="sourceLineNo">923</span> LOG.debug("Waited {} ms, but {} is still running, skipping bypass with force={}",<a name="line.923"></a>
-<span class="sourceLineNo">924</span> lockWait, procedure, override);<a name="line.924"></a>
-<span class="sourceLineNo">925</span> return false;<a name="line.925"></a>
-<span class="sourceLineNo">926</span> } else if (lockEntry == null) {<a name="line.926"></a>
-<span class="sourceLineNo">927</span> LOG.debug("Waited {} ms, but {} is still running, begin bypass with force={}",<a name="line.927"></a>
-<span class="sourceLineNo">928</span> lockWait, procedure, override);<a name="line.928"></a>
-<span class="sourceLineNo">929</span> }<a name="line.929"></a>
-<span class="sourceLineNo">930</span> try {<a name="line.930"></a>
-<span class="sourceLineNo">931</span> // check whether the procedure is already finished<a name="line.931"></a>
-<span class="sourceLineNo">932</span> if (procedure.isFinished()) {<a name="line.932"></a>
-<span class="sourceLineNo">933</span> LOG.debug("{} is already finished, skipping bypass", procedure);<a name="line.933"></a>
-<span class="sourceLineNo">934</span> return false;<a name="line.934"></a>
-<span class="sourceLineNo">935</span> }<a name="line.935"></a>
-<span class="sourceLineNo">936</span><a name="line.936"></a>
-<span class="sourceLineNo">937</span> if (procedure.hasChildren()) {<a name="line.937"></a>
-<span class="sourceLineNo">938</span> if (recursive) {<a name="line.938"></a>
-<span class="sourceLineNo">939</span> // EXPENSIVE. Checks each live procedure of which there could be many!!!<a name="line.939"></a>
-<span class="sourceLineNo">940</span> // Is there another way to get children of a procedure?<a name="line.940"></a>
-<span class="sourceLineNo">941</span> LOG.info("Recursive bypass on children of pid={}", procedure.getProcId());<a name="line.941"></a>
-<span class="sourceLineNo">942</span> this.procedures.forEachValue(1 /*Single-threaded*/,<a name="line.942"></a>
-<span class="sourceLineNo">943</span> // Transformer<a name="line.943"></a>
-<span class="sourceLineNo">944</span> v -> v.getParentProcId() == procedure.getProcId()? v: null,<a name="line.944"></a>
-<span class="sourceLineNo">945</span> // Consumer<a name="line.945"></a>
-<span class="sourceLineNo">946</span> v -> {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> try {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> bypassProcedure(v.getProcId(), lockWait, override, recursive);<a name="line.948"></a>
-<span class="sourceLineNo">949</span> } catch (IOException e) {<a name="line.949"></a>
-<span class="sourceLineNo">950</span> LOG.warn("Recursive bypass of pid={}", v.getProcId(), e);<a name="line.950"></a>
-<span class="sourceLineNo">951</span> }<a name="line.951"></a>
-<span class="sourceLineNo">952</span> });<a name="line.952"></a>
-<span class="sourceLineNo">953</span> } else {<a name="line.953"></a>
-<span class="sourceLineNo">954</span> LOG.debug("{} has children, skipping bypass", procedure);<a name="line.954"></a>
-<span class="sourceLineNo">955</span> return false;<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span><a name="line.958"></a>
-<span class="sourceLineNo">959</span> // If the procedure has no parent or no child, we are safe to bypass it in whatever state<a name="line.959"></a>
-<span class="sourceLineNo">960</span> if (procedure.hasParent() && procedure.getState() != ProcedureState.RUNNABLE<a name="line.960"></a>
-<span class="sourceLineNo">961</span> && procedure.getState() != ProcedureState.WAITING<a name="line.961"></a>
-<span class="sourceLineNo">962</span> && procedure.getState() != ProcedureState.WAITING_TIMEOUT) {<a name="line.962"></a>
-<span class="sourceLineNo">963</span> LOG.debug("Bypassing procedures in RUNNABLE, WAITING and WAITING_TIMEOUT states "<a name="line.963"></a>
-<span class="sourceLineNo">964</span> + "(with no parent), {}",<a name="line.964"></a>
-<span class="sourceLineNo">965</span> procedure);<a name="line.965"></a>
-<span class="sourceLineNo">966</span> // Question: how is the bypass done here?<a name="line.966"></a>
-<span class="sourceLineNo">967</span> return false;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> }<a name="line.968"></a>
-<span class="sourceLineNo">969</span><a name="line.969"></a>
-<span class="sourceLineNo">970</span> // Now, the procedure is not finished, and no one can execute it since we take the lock now<a name="line.970"></a>
-<span class="sourceLineNo">971</span> // And we can be sure that its ancestor is not running too, since their child has not<a name="line.971"></a>
-<span class="sourceLineNo">972</span> // finished yet<a name="line.972"></a>
-<span class="sourceLineNo">973</span> Procedure<TEnvironment> current = procedure;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> while (current != null) {<a name="line.974"></a>
-<span class="sourceLineNo">975</span> LOG.debug("Bypassing {}", current);<a name="line.975"></a>
-<span class="sourceLineNo">976</span> current.bypass(getEnvironment());<a name="line.976"></a>
-<span class="sourceLineNo">977</span> store.update(procedure);<a name="line.977"></a>
-<span class="sourceLineNo">978</span> long parentID = current.getParentProcId();<a name="line.978"></a>
-<span class="sourceLineNo">979</span> current = getProcedure(parentID);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> }<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span> //wake up waiting procedure, already checked there is no child<a name="line.982"></a>
-<span class="sourceLineNo">983</span> if (procedure.getState() == ProcedureState.WAITING) {<a name="line.983"></a>
-<span class="sourceLineNo">984</span> procedure.setState(ProcedureState.RUNNABLE);<a name="line.984"></a>
-<span class="sourceLineNo">985</span> store.update(procedure);<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> // If state of procedure is WAITING_TIMEOUT, we can directly submit it to the scheduler.<a name="line.988"></a>
-<span class="sourceLineNo">989</span> // Instead we should remove it from timeout Executor queue and tranfer its state to RUNNABLE<a name="line.989"></a>
-<span class="sourceLineNo">990</span> if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.990"></a>
-<span class="sourceLineNo">991</span> LOG.debug("transform procedure {} from WAITING_TIMEOUT to RUNNABLE", procedure);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> if (timeoutExecutor.remove(procedure)) {<a name="line.992"></a>
-<span class="sourceLineNo">993</span> LOG.debug("removed procedure {} from timeoutExecutor", procedure);<a name="line.993"></a>
-<span class="sourceLineNo">994</span> timeoutExecutor.executeTimedoutProcedure(procedure);<a name="line.994"></a>
-<span class="sourceLineNo">995</span> }<a name="line.995"></a>
-<span class="sourceLineNo">996</span> } else if (lockEntry != null) {<a name="line.996"></a>
-<span class="sourceLineNo">997</span> scheduler.addFront(procedure);<a name="line.997"></a>
-<span class="sourceLineNo">998</span> LOG.debug("Bypassing {} and its ancestors successfully, adding to queue", procedure);<a name="line.998"></a>
-<span class="sourceLineNo">999</span> } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> // If we don't have the lock, we can't re-submit the queue,<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> // since it is already executing. To get rid of the stuck situation, we<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span> // need to restart the master. With the procedure set to bypass, the procedureExecutor<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> // will bypass it and won't get stuck again.<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> LOG.debug("Bypassing {} and its ancestors successfully, but since it is already running, "<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> + "skipping add to queue",<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span> procedure);<a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return true;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span><a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> } finally {<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span> if (lockEntry != null) {<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> }<a name="line.1015"></a>
+<span class="sourceLineNo">882</span> * A procedure can be bypassed only if<a name="line.882"></a>
+<span class="sourceLineNo">883</span> * 1. The procedure is in state of RUNNABLE, WAITING, WAITING_TIMEOUT<a name="line.883"></a>
+<span class="sourceLineNo">884</span> * or it is a root procedure without any child.<a name="line.884"></a>
+<span class="sourceLineNo">885</span> * 2. No other worker thread is executing it<a name="line.885"></a>
+<span class="sourceLineNo">886</span> * 3. No child procedure has been submitted<a name="line.886"></a>
+<span class="sourceLineNo">887</span> *<a name="line.887"></a>
+<span class="sourceLineNo">888</span> * <p><a name="line.888"></a>
+<span class="sourceLineNo">889</span> * If all the requirements are meet, the procedure and its ancestors will be<a name="line.889"></a>
+<span class="sourceLineNo">890</span> * bypassed and persisted to WAL.<a name="line.890"></a>
+<span class="sourceLineNo">891</span> *<a name="line.891"></a>
+<span class="sourceLineNo">892</span> * <p><a name="line.892"></a>
+<span class="sourceLineNo">893</span> * If the procedure is in WAITING state, will set it to RUNNABLE add it to run queue.<a name="line.893"></a>
+<span class="sourceLineNo">894</span> * TODO: What about WAITING_TIMEOUT?<a name="line.894"></a>
+<span class="sourceLineNo">895</span> * @param pids the procedure id<a name="line.895"></a>
+<span class="sourceLineNo">896</span> * @param lockWait time to wait lock<a name="line.896"></a>
+<span class="sourceLineNo">897</span> * @param force if force set to true, we will bypass the procedure even if it is executing.<a name="line.897"></a>
+<span class="sourceLineNo">898</span> * This is for procedures which can't break out during executing(due to bug, mostly)<a name="line.898"></a>
+<span class="sourceLineNo">899</span> * In this case, bypassing the procedure is not enough, since it is already stuck<a name="line.899"></a>
+<span class="sourceLineNo">900</span> * there. We need to restart the master after bypassing, and letting the problematic<a name="line.900"></a>
+<span class="sourceLineNo">901</span> * procedure to execute wth bypass=true, so in that condition, the procedure can be<a name="line.901"></a>
+<span class="sourceLineNo">902</span> * successfully bypassed.<a name="line.902"></a>
+<span class="sourceLineNo">903</span> * @param recursive We will do an expensive search for children of each pid. EXPENSIVE!<a name="line.903"></a>
+<span class="sourceLineNo">904</span> * @return true if bypass success<a name="line.904"></a>
+<span class="sourceLineNo">905</span> * @throws IOException IOException<a name="line.905"></a>
+<span class="sourceLineNo">906</span> */<a name="line.906"></a>
+<span class="sourceLineNo">907</span> public List<Boolean> bypassProcedure(List<Long> pids, long lockWait, boolean force,<a name="line.907"></a>
+<span class="sourceLineNo">908</span> boolean recursive)<a name="line.908"></a>
+<span class="sourceLineNo">909</span> throws IOException {<a name="line.909"></a>
+<span class="sourceLineNo">910</span> List<Boolean> result = new ArrayList<Boolean>(pids.size());<a name="line.910"></a>
+<span class="sourceLineNo">911</span> for(long pid: pids) {<a name="line.911"></a>
+<span class="sourceLineNo">912</span> result.add(bypassProcedure(pid, lockWait, force, recursive));<a name="line.912"></a>
+<span class="sourceLineNo">913</span> }<a name="line.913"></a>
+<span class="sourceLineNo">914</span> return result;<a name="line.914"></a>
+<span class="sourceLineNo">915</span> }<a name="line.915"></a>
+<span class="sourceLineNo">916</span><a name="line.916"></a>
+<span class="sourceLineNo">917</span> boolean bypassProcedure(long pid, long lockWait, boolean override, boolean recursive)<a name="line.917"></a>
+<span class="sourceLineNo">918</span> throws IOException {<a name="line.918"></a>
+<span class="sourceLineNo">919</span> Preconditions.checkArgument(lockWait > 0, "lockWait should be positive");<a name="line.919"></a>
+<span class="sourceLineNo">920</span> final Procedure<TEnvironment> procedure = getProcedure(pid);<a name="line.920"></a>
+<span class="sourceLineNo">921</span> if (procedure == null) {<a name="line.921"></a>
+<span class="sourceLineNo">922</span> LOG.debug("Procedure pid={} does not exist, skipping bypass", pid);<a name="line.922"></a>
+<span class="sourceLineNo">923</span> return false;<a name="line.923"></a>
+<span class="sourceLineNo">924</span> }<a name="line.924"></a>
+<span class="sourceLineNo">925</span><a name="line.925"></a>
+<span class="sourceLineNo">926</span> LOG.debug("Begin bypass {} with lockWait={}, override={}, recursive={}",<a name="line.926"></a>
+<span class="sourceLineNo">927</span> procedure, lockWait, override, recursive);<a name="line.927"></a>
+<span class="sourceLineNo">928</span> IdLock.Entry lockEntry = procExecutionLock.tryLockEntry(procedure.getProcId(), lockWait);<a name="line.928"></a>
+<span class="sourceLineNo">929</span> if (lockEntry == null && !override) {<a name="line.929"></a>
+<span class="sourceLineNo">930</span> LOG.debug("Waited {} ms, but {} is still running, skipping bypass with force={}",<a name="line.930"></a>
+<span class="sourceLineNo">931</span> lockWait, procedure, override);<a name="line.931"></a>
+<span class="sourceLineNo">932</span> return false;<a name="line.932"></a>
+<span class="sourceLineNo">933</span> } else if (lockEntry == null) {<a name="line.933"></a>
+<span class="sourceLineNo">934</span> LOG.debug("Waited {} ms, but {} is still running, begin bypass with force={}",<a name="line.934"></a>
+<span class="sourceLineNo">935</span> lockWait, procedure, override);<a name="line.935"></a>
+<span class="sourceLineNo">936</span> }<a name="line.936"></a>
+<span class="sourceLineNo">937</span> try {<a name="line.937"></a>
+<span class="sourceLineNo">938</span> // check whether the procedure is already finished<a name="line.938"></a>
+<span class="sourceLineNo">939</span> if (procedure.isFinished()) {<a name="line.939"></a>
+<span class="sourceLineNo">940</span> LOG.debug("{} is already finished, skipping bypass", procedure);<a name="line.940"></a>
+<span class="sourceLineNo">941</span> return false;<a name="line.941"></a>
+<span class="sourceLineNo">942</span> }<a name="line.942"></a>
+<span class="sourceLineNo">943</span><a name="line.943"></a>
+<span class="sourceLineNo">944</span> if (procedure.hasChildren()) {<a name="line.944"></a>
+<span class="sourceLineNo">945</span> if (recursive) {<a name="line.945"></a>
+<span class="sourceLineNo">946</span> // EXPENSIVE. Checks each live procedure of which there could be many!!!<a name="line.946"></a>
+<span class="sourceLineNo">947</span> // Is there another way to get children of a procedure?<a name="line.947"></a>
+<span class="sourceLineNo">948</span> LOG.info("Recursive bypass on children of pid={}", procedure.getProcId());<a name="line.948"></a>
+<span class="sourceLineNo">949</span> this.procedures.forEachValue(1 /*Single-threaded*/,<a name="line.949"></a>
+<span class="sourceLineNo">950</span> // Transformer<a name="line.950"></a>
+<span class="sourceLineNo">951</span> v -> v.getParentProcId() == procedure.getProcId()? v: null,<a name="line.951"></a>
+<span class="sourceLineNo">952</span> // Consumer<a name="line.952"></a>
+<span class="sourceLineNo">953</span> v -> {<a name="line.953"></a>
+<span class="sourceLineNo">954</span> try {<a name="line.954"></a>
+<span class="sourceLineNo">955</span> bypassProcedure(v.getProcId(), lockWait, override, recursive);<a name="line.955"></a>
+<span class="sourceLineNo">956</span> } catch (IOException e) {<a name="line.956"></a>
+<span class="sourceLineNo">957</span> LOG.warn("Recursive bypass of pid={}", v.getProcId(), e);<a name="line.957"></a>
+<span class="sourceLineNo">958</span> }<a name="line.958"></a>
+<span class="sourceLineNo">959</span> });<a name="line.959"></a>
+<span class="sourceLineNo">960</span> } else {<a name="line.960"></a>
+<span class="sourceLineNo">961</span> LOG.debug("{} has children, skipping bypass", procedure);<a name="line.961"></a>
+<span class="sourceLineNo">962</span> return false;<a name="line.962"></a>
+<span class="sourceLineNo">963</span> }<a name="line.963"></a>
+<span class="sourceLineNo">964</span> }<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span> // If the procedure has no parent or no child, we are safe to bypass it in whatever state<a name="line.966"></a>
+<span class="sourceLineNo">967</span> if (procedure.hasParent() && procedure.getState() != ProcedureState.RUNNABLE<a name="line.967"></a>
+<span class="sourceLineNo">968</span> && procedure.getState() != ProcedureState.WAITING<a name="line.968"></a>
+<span class="sourceLineNo">969</span> && procedure.getState() != ProcedureState.WAITING_TIMEOUT) {<a name="line.969"></a>
+<span class="sourceLineNo">970</span> LOG.debug("Bypassing procedures in RUNNABLE, WAITING and WAITING_TIMEOUT states "<a name="line.970"></a>
+<span class="sourceLineNo">971</span> + "(with no parent), {}",<a name="line.971"></a>
+<span class="sourceLineNo">972</span> procedure);<a name="line.972"></a>
+<span class="sourceLineNo">973</span> // Question: how is the bypass done here?<a name="line.973"></a>
+<span class="sourceLineNo">974</span> return false;<a name="line.974"></a>
+<span class="sourceLineNo">975</span> }<a name="line.975"></a>
+<span class="sourceLineNo">976</span><a name="line.976"></a>
+<span class="sourceLineNo">977</span> // Now, the procedure is not finished, and no one can execute it since we take the lock now<a name="line.977"></a>
+<span class="sourceLineNo">978</span> // And we can be sure that its ancestor is not running too, since their child has not<a name="line.978"></a>
+<span class="sourceLineNo">979</span> // finished yet<a name="line.979"></a>
+<span class="sourceLineNo">980</span> Procedure<TEnvironment> current = procedure;<a name="line.980"></a>
+<span class="sourceLineNo">981</span> while (current != null) {<a name="line.981"></a>
+<span class="sourceLineNo">982</span> LOG.debug("Bypassing {}", current);<a name="line.982"></a>
+<span class="sourceLineNo">983</span> current.bypass(getEnvironment());<a name="line.983"></a>
+<span class="sourceLineNo">984</span> store.update(procedure);<a name="line.984"></a>
+<span class="sourceLineNo">985</span> long parentID = current.getParentProcId();<a name="line.985"></a>
+<span class="sourceLineNo">986</span> current = getProcedure(parentID);<a name="line.986"></a>
+<span class="sourceLineNo">987</span> }<a name="line.987"></a>
+<span class="sourceLineNo">988</span><a name="line.988"></a>
+<span class="sourceLineNo">989</span> //wake up waiting procedure, already checked there is no child<a name="line.989"></a>
+<span class="sourceLineNo">990</span> if (procedure.getState() == ProcedureState.WAITING) {<a name="line.990"></a>
+<span class="sourceLineNo">991</span> procedure.setState(ProcedureState.RUNNABLE);<a name="line.991"></a>
+<span class="sourceLineNo">992</span> store.update(procedure);<a name="line.992"></a>
+<span class="sourceLineNo">993</span> }<a name="line.993"></a>
+<span class="sourceLineNo">994</span><a name="line.994"></a>
+<span class="sourceLineNo">995</span> // If state of procedure is WAITING_TIMEOUT, we can directly submit it to the scheduler.<a name="line.995"></a>
+<span class="sourceLineNo">996</span> // Instead we should remove it from timeout Executor queue and tranfer its state to RUNNABLE<a name="line.996"></a>
+<span class="sourceLineNo">997</span> if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span> LOG.debug("transform procedure {} from WAITING_TIMEOUT to RUNNABLE", procedure);<a name="line.998"></a>
+<span class="sourceLineNo">999</span> if (timeoutExecutor.remove(procedure)) {<a name="line.999"></a>
+<span class="sourceLineNo">1000</span> LOG.debug("removed procedure {} from timeoutExecutor", procedure);<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span> timeoutExecutor.executeTimedoutProcedure(procedure);<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span> }<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span> } else if (lockEntry != null) {<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span> scheduler.addFront(procedure);<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span> LOG.debug("Bypassing {} and its ancestors successfully, adding to queue", procedure);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span> } else {<a name="line.1006"></a>
+<span class="sourceLineNo">1007</span> // If we don't have the lock, we can't re-submit the queue,<a name="line.1007"></a>
+<span class="sourceLineNo">1008</span> // since it is already executing. To get rid of the stuck situation, we<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span> // need to restart the master. With the procedure set to bypass, the procedureExecutor<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span> // will bypass it and won't get stuck again.<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span> LOG.debug("Bypassing {} and its ancestors successfully, but since it is already running, "<a name="line.1011"></a>
+<span class="sourceLineNo">1012</span> + "skipping add to queue",<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span> procedure);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span> return true;<a name="line.1015"></a>
<span class="sourceLineNo">1016</span><a name="line.1016"></a>
-<span class="sourceLineNo">1017</span> /**<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> * Add a new root-procedure to the executor.<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> * @param proc the new procedure to execute.<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> * @param nonceKey the registered unique identifier for this operation from the client or process.<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> * @return the procedure id, that can be used to monitor the operation<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> */<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span> justification = "FindBugs is blind to the check-for-null")<a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> public long submitProcedure(Procedure<TEnvironment> proc, NonceKey nonceKey) {<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span><a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> prepareProcedure(proc);<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span><a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> final Long currentProcId;<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> if (nonceKey != null) {<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> currentProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> Preconditions.checkArgument(currentProcId != null,<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span> "Expected nonceKey=" + nonceKey + " to be reserved, use registerNonce(); proc=" + proc);<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> } else {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> currentProcId = nextProcId();<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> }<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span><a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> // Initialize the procedure<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> proc.setNonceKey(nonceKey);<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> proc.setProcId(currentProcId.longValue());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span><a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> // Commit the transaction<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> store.insert(proc, null);<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> LOG.debug("Stored {}", proc);<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span><a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> // Add the procedure to the executor<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> return pushProcedure(proc);<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> }<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span><a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> /**<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> * Add a set of new root-procedure to the executor.<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> * @param procs the new procedures to execute.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> */<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span> // TODO: Do we need to take nonces here?<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> public void submitProcedures(Procedure<TEnvironment>[] procs) {<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> if (procs == null || procs.length <= 0) {<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> return;<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> }<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span><a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> // Prepare procedure<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span> prepareProcedure(procs[i]).setProcId(nextProcId());<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> }<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span><a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> // Commit the transaction<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span> store.insert(procs);<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> if (LOG.isDebugEnabled()) {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> LOG.debug("Stored " + Arrays.toString(procs));<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> }<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span><a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> // Add the procedure to the executor<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> pushProcedure(procs[i]);<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span> }<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> }<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span><a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> private Procedure<TEnvironment> prepareProcedure(Procedure<TEnvironment> proc) {<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> Preconditions.checkArgument(proc.getState() == ProcedureState.INITIALIZING);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> Preconditions.checkArgument(!proc.hasParent(), "unexpected parent", proc);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span> if (this.checkOwnerSet) {<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> Preconditions.checkArgument(proc.hasOwner(), "missing owner");<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> }<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span> return proc;<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span><a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> private long pushProcedure(Procedure<TEnvironment> proc) {<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span> final long currentProcId = proc.getProcId();<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span><a name="line.1090"></a>
-<span class="sourceLineNo">1091</span> // Update metrics on start of a procedure<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span> proc.updateMetricsOnSubmit(getEnvironment());<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span><a name="line.1093"></a>
-<span class="sourceLineNo">1094</span> // Create the rollback stack for the procedure<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span> RootProcedureState<TEnvironment> stack = new RootProcedureState<>();<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span> rollbackStack.put(currentProcId, stack);<a name="line.1096"></a>
+<span class="sourceLineNo">1017</span> } finally {<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span> if (lockEntry != null) {<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span> }<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span> }<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span> }<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span><a name="line.1023"></a>
+<span class="sourceLineNo">1024</span> /**<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span> * Add a new root-procedure to the executor.<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span> * @param proc the new procedure to execute.<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span> * @param nonceKey the registered unique identifier for this operation from the client or process.<a name="line.1027"></a>
+<span class="sourceLineNo">1028</span> * @return the procedure id, that can be used to monitor the operation<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span> */<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span> justification = "FindBugs is blind to the check-for-null")<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span> public long submitProcedure(Procedure<TEnvironment> proc, NonceKey nonceKey) {<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span><a name="line.1034"></a>
+<span class="sourceLineNo">1035</span> prepareProcedure(proc);<a name="line.1035"></a>
+<span class="sourceLineNo">1036</span><a name="line.1036"></a>
+<span class="sourceLineNo">1037</span> final Long currentProcId;<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span> if (nonceKey != null) {<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span> currentProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span> Preconditions.checkArgument(currentProcId != null,<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span> "Expected nonceKey=" + nonceKey + " to be reserved, use registerNonce(); proc=" + proc);<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span> } else {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span> currentProcId = nextProcId();<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span><a name="line.1045"></a>
+<span class="sourceLineNo">1046</span> // Initialize the procedure<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span> proc.setNonceKey(nonceKey);<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span> proc.setProcId(currentProcId.longValue());<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span><a name="line.1049"></a>
+<span class="sourceLineNo">1050</span> // Commit the transaction<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span> store.insert(proc, null);<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span> LOG.debug("Stored {}", proc);<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span><a name="line.1053"></a>
+<span class="sourceLineNo">1054</span> // Add the procedure to the executor<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span> return pushProcedure(proc);<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span> }<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span><a name="line.1057"></a>
+<span class="sourceLineNo">1058</span> /**<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span> * Add a set of new root-procedure to the executor.<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span> * @param procs the new procedures to execute.<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span> */<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span> // TODO: Do we need to take nonces here?<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span> public void submitProcedures(Procedure<TEnvironment>[] procs) {<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span> if (procs == null || procs.length <= 0) {<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span> return;<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span><a name="line.1068"></a>
+<span class="sourceLineNo">1069</span> // Prepare procedure<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span> prepareProcedure(procs[i]).setProcId(nextProcId());<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span> }<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span><a name="line.1073"></a>
+<span class="sourceLineNo">1074</span> // Commit the transaction<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span> store.insert(procs);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span> if (LOG.isDebugEnabled()) {<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span> LOG.debug("Stored " + Arrays.toString(procs));<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span> }<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span><a name="line.1079"></a>
+<span class="sourceLineNo">1080</span> // Add the procedure to the executor<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span> pushProcedure(procs[i]);<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span> }<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span> }<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span><a name="line.1085"></a>
+<span class="sourceLineNo">1086</span> private Procedure<TEnvironment> prepareProcedure(Procedure<TEnvironment> proc) {<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span> Preconditions.checkArgument(proc.getState() == ProcedureState.INITIALIZING);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span> Preconditions.checkArgument(!proc.hasParent(), "unexpected parent", proc);<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span> if (this.checkOwnerSet) {<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span> Preconditions.checkArgument(proc.hasOwner(), "missing owner");<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span> }<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span> return proc;<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span> }<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span><a name="line.1094"></a>
+<span class="sourceLineNo">1095</span> private long pushProcedure(Procedure<TEnvironment> proc) {<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span> final long currentProcId = proc.getProcId();<a name="line.1096"></a>
<span class="sourceLineNo">1097</span><a name="line.1097"></a>
-<span class="sourceLineNo">1098</span> // Submit the new subprocedures<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span> assert !procedures.containsKey(currentProcId);<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span> procedures.put(currentProcId, proc);<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span> sendProcedureAddedNotification(currentProcId);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span> scheduler.addBack(proc);<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span> return proc.getProcId();<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span> }<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span><a name="line.1105"></a>
-<span class="sourceLineNo">1106</span> /**<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span> * Send an abort notification the specified procedure.<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span> * Depending on the procedure implementation the abort can be considered or ignored.<a name="line.1108"></a>
-<span class="sourceLineNo">1109</span> * @param procId the procedure to abort<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span> */<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span> public boolean abort(long procId) {<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span> return abort(procId, true);<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span> }<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span> /**<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span> * Send an abort notification to the specified procedure.<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span> * Depending on the procedure implementation, the abort can be considered or ignored.<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span> * @param procId the procedure to abort<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span> * @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span> */<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span> public boolean abort(long procId, boolean mayInterruptIfRunning) {<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span> if (proc != null) {<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span> if (!mayInterruptIfRunning && proc.wasExecuted()) {<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span> return false;<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span> }<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span> return proc.abort(getEnvironment());<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span> }<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span> return false;<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span> }<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span><a name="line.1133"></a>
-<span class="sourceLineNo">1134</span> // ==========================================================================<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span> // Executor query helpers<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span> // ==========================================================================<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span> public Procedure<TEnvironment> getProcedure(final long procId) {<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span> return procedures.get(procId);<a name="line.1138"></a>
+<span class="sourceLineNo">1098</span> // Update metrics on start of a procedure<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span> proc.updateMetricsOnSubmit(getEnvironment());<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span><a name="line.1100"></a>
+<span class="sourceLineNo">1101</span> // Create the rollback stack for the procedure<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span> RootProcedureState<TEnvironment> stack = new RootProcedureState<>();<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span> rollbackStack.put(currentProcId, stack);<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span><a name="line.1104"></a>
+<span class="sourceLineNo">1105</span> // Submit the new subprocedures<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span> assert !procedures.containsKey(currentProcId);<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span> procedures.put(currentProcId, proc);<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span> sendProcedureAddedNotification(currentProcId);<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span> scheduler.addBack(proc);<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span> return proc.getProcId();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span> }<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span><a name="line.1112"></a>
+<span class="sourceLineNo">1113</span> /**<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span> * Send an abort notification the specified procedure.<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span> * Depending on the procedure implementation the abort can be considered or ignored.<a name="line.1115"></a>
+<span class="sourceLineNo">1116</span> * @param procId the procedure to abort<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span> */<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span> public boolean abort(long procId) {<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span> return abort(procId, true);<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span> }<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span><a name="line.1122"></a>
+<span class="sourceLineNo">1123</span> /**<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span> * Send an abort notification to the specified procedure.<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span> * Depending on the procedure implementation, the abort can be considered or ignored.<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span> * @param procId the procedure to abort<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span> * @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span> */<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span> public boolean abort(long procId, boolean mayInterruptIfRunning) {<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span> if (proc != null) {<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span> if (!mayInterruptIfRunning && proc.wasExecuted()) {<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span> return false;<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span> }<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span> return proc.abort(getEnvironment());<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span> }<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span> return false;<a name="line.1138"></a>
<span class="sourceLineNo">1139</span> }<a name="line.1139"></a>
<span class="sourceLineNo">1140</span><a name="line.1140"></a>
-<span class="sourceLineNo">1141</span> public <T extends Procedure<TEnvironment>> T getProcedure(Class<T> clazz, long procId) {<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span> Procedure<TEnvironment> proc = getProcedure(procId);<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span> if (clazz.isInstance(proc)) {<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span> return clazz.cast(proc);<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span> }<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span> return null;<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span> }<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span><a name="line.1148"></a>
-<span class="sourceLineNo">1149</span> public Procedure<TEnvironment> getResult(long procId) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span> if (retainer == null) {<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span> return null;<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span> } else {<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span> return retainer.getProcedure();<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span> }<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span> }<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span><a name="line.1157"></a>
-<span class="sourceLineNo">1158</span> /**<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span> * Return true if the procedure is finished.<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span> * The state may be "completed successfully" or "failed and rolledback".<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span> * Use getResult() to check the state or get the result data.<a name="line.1161"></a>
-<span class="sourceLineNo">1162</span> * @param procId the ID of the procedure to check<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span> * @return true if the procedure execution is finished, otherwise false.<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span> */<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span> public boolean isFinished(final long procId) {<a name="line.1165"></a>
-<span class="sourceLineNo">1166</span> return !procedures.containsKey(procId);<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span> }<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span><a name="line.1168"></a>
-<span class="sourceLineNo">1169</span> /**<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span> * Return true if the procedure is started.<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span> * @param procId the ID of the procedure to check<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span> * @return true if the procedure execution is started, otherwise false.<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span> */<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span> public boolean isStarted(long procId) {<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span> Procedure<?> proc = procedures.get(procId);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span> if (proc == null) {<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span> return completed.get(procId) != null;<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span> }<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span> return proc.wasExecuted();<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span> }<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span><a name="line.1181"></a>
-<span class="sourceLineNo">1182</span> /**<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span> * Mark the specified completed procedure, as ready to remove.<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span> * @param procId the ID of the procedure to remove<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span> */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span> public void removeResult(long procId) {<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span> if (retainer == null) {<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span> assert !procedures.containsKey(procId) : "pid=" + procId + " is still running";<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span> LOG.debug("pid={} already removed by the cleaner.", procId);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span> return;<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span> }<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span><a name="line.1193"></a>
-<span class="sourceLineNo">1194</span> // The CompletedProcedureCleaner will take care of deletion, once the TTL is expired.<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span> retainer.setClientAckTime(EnvironmentEdgeManager.currentTime());<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span> }<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span><a name="line.1197"></a>
-<span class="sourceLineNo">1198</span> public Procedure<TEnvironment> getResultOrProcedure(long procId) {<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span> if (retainer == null) {<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span> return procedures.get(procId);<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span> } else {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span> return retainer.getProcedure();<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span> }<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span> }<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span><a name="line.1206"></a>
-<span class="sourceLineNo">1207</span> /**<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span> * Check if the user is this procedure's owner<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span> * @param procId the target procedure<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span> * @param user the user<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span> * @return true if the user is the owner of the procedure,<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span> * false otherwise or the owner is unknown.<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span> */<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span> public boolean isProcedureOwner(long procId, User user) {<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span> if (user == null) {<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span> return false;<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span> }<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span> final Procedure<TEnvironment> runningProc = procedures.get(procId);<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span> if (runningProc != null) {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span> return runningProc.getOwner().equals(user.getShortName());<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span> }<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span><a name="line.1222"></a>
-<span class="sourceLineNo">1223</span> final CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span> if (retainer != null) {<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span> return retainer.getProcedure().getOwner().equals(user.getShortName());<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span> }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span> // Procedure either does not exist or has already completed and got cleaned up.<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span> // At this time, we cannot check the owner of the procedure<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span> return false;<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span> }<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span><a name="line.1232"></a>
-<span class="sourceLineNo">1233</span> /**<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span> * Should only be used when starting up, where the procedure workers have not been started.<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span> * <p/><a name="line.1235"></a>
-<span class="sourceLineNo">1236</span> * If the procedure works has been started, the return values maybe changed when you are<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span> * processing it so usually this is not safe. Use {@link #getProcedures()} below for most cases as<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span> * it will do a copy, and also include the finished procedures.<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span> */<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span> public Collection<Procedure<TEnvironment>> getActiveProceduresNoCopy() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span> return procedures.values();<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span> }<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span><a name="line.1243"></a>
-<span class="sourceLineNo">1244</span> /**<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span> * Get procedures.<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span> * @return the procedures in a list<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span> */<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span> public List<Procedure<TEnvironment>> getProcedures() {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span> List<Procedure<TEnvironment>> procedureList =<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span> new ArrayList<>(procedures.size() + completed.size());<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span> procedureList.addAll(procedures.values());<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span> // Note: The procedure could show up twice in the list with different state, as<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span> // it could complete after we walk through procedures list and insert into<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span> // procedureList - it is ok, as we will use the information in the Procedure<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span> // to figure it out; to prevent this would increase the complexity of the logic.<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span> completed.values().stream().map(CompletedProcedureRetainer::getProcedure)<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span> .forEach(procedureList::add);<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span> return procedureList;<a name="line.1258"></a>
-<span class="sourceLineNo">1259</span> }<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span><a name="line.1260"></a>
-<span class="sourceLineNo">1261</span> // ==========================================================================<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span> // Listeners helpers<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span> // ==========================================================================<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span> public void registerListener(ProcedureExecutorListener listener) {<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span> this.listeners.add(listener);<a name="line.1265"></a>
+<span class="sourceLineNo">1141</span> // ==========================================================================<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span> // Executor query helpers<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span> // ==========================================================================<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span> public Procedure<TEnvironment> getProcedure(final long procId) {<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span> return procedures.get(procId);<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span> }<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span><a name="line.1147"></a>
+<span class="sourceLineNo">1148</span> public <T extends Procedure<TEnvironment>> T getProcedure(Class<T> clazz, long procId) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span> Procedure<TEnvironment> proc = getProcedure(procId);<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span> if (clazz.isInstance(proc)) {<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span> return clazz.cast(proc);<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span> }<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span> return null;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span> }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span><a name="line.1155"></a>
+<span class="sourceLineNo">1156</span> public Procedure<TEnvironment> getResult(long procId) {<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span> if (retainer == null) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span> return null;<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span> } else {<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span> return retainer.getProcedure();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span> }<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span> }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span> /**<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span> * Return true if the procedure is finished.<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span> * The state may be "completed successfully" or "failed and rolledback".<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span> * Use getResult() to check the state or get the result data.<a name="line.1168"></a>
+<span class="sourceLineNo">1169</span> * @param procId the ID of the procedure to check<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span> * @return true if the procedure execution is finished, otherwise false.<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span> */<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span> public boolean isFinished(final long procId) {<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span> return !procedures.containsKey(procId);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span> }<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span><a name="line.1175"></a>
+<span class="sourceLineNo">1176</span> /**<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span> * Return true if the procedure is started.<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span> * @param procId the ID of the procedure to check<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span> * @return true if the procedure execution is started, otherwise false.<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span> */<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span> public boolean isStarted(long procId) {<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span> Procedure<?> proc = procedures.get(procId);<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span> if (proc == null) {<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span> return completed.get(procId) != null;<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span> return proc.wasExecuted();<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span> }<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span><a name="line.1188"></a>
+<span class="sourceLineNo">1189</span> /**<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span> * Mark the specified completed procedure, as ready to remove.<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span> * @param procId the ID of the procedure to remove<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span> */<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span> public void removeResult(long procId) {<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span> if (retainer == null) {<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span> assert !procedures.containsKey(procId) : "pid=" + procId + " is still running";<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span> LOG.debug("pid={} already removed by the cleaner.", procId);<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span> return;<a name="line.1198"></a>
+<span class="sourceLineNo">1199</span> }<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span><a name="line.1200"></a>
+<span class="sourceLineNo">1201</span> // The CompletedProcedureCleaner will take care of deletion, once the TTL is expired.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span> retainer.setClientAckTime(EnvironmentEdgeManager.currentTime());<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span> }<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span><a name="line.1204"></a>
+<span class="sourceLineNo">1205</span> public Procedure<TEnvironment> getResultOrProcedure(long procId) {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span> if (retainer == null) {<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span> return procedures.get(procId);<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span> } else {<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span> return retainer.getProcedure();<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span> }<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span> }<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span><a name="line.1213"></a>
+<span class="sourceLineNo">1214</span> /**<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span> * Check if the user is this procedure's owner<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span> * @param procId the target procedure<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span> * @param user the user<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span> * @return true if the user is the owner of the procedure,<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span> * false otherwise or the owner is unknown.<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span> */<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span> public boolean isProcedureOwner(long procId, User user) {<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span> if (user == null) {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span> return false;<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span> }<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span> final Procedure<TEnvironment> runningProc = procedures.get(procId);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span> if (runningProc != null) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span> return runningProc.getOwner().equals(user.getShortName());<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span> }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span><a name="line.1229"></a>
+<span class="sourceLineNo">1230</span> final CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span> if (retainer != null) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span> return retainer.getProcedure().getOwner().equals(user.getShortName());<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span> }<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span><a name="line.1234"></a>
+<span class="sourceLineNo">1235</span> // Procedure either does not exist or has already completed and got cleaned up.<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span> // At this time, we cannot check the owner of the procedure<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span> return false;<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span> }<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span><a name="line.1239"></a>
+<span class="sourceLineNo">1240</span> /**<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span> * Should only be used when starting up, where the procedure workers have not been started.<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span> * <p/><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span> * If the procedure works has been started, the return values maybe changed when you are<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span> * processing it so usually this is not safe. Use {@link #getProcedures()} below for most cases as<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span> * it will do a copy, and also include the finished procedures.<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span> */<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span> public Collection<Procedure<TEnvironment>> getActiveProceduresNoCopy() {<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span> return procedures.values();<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span> }<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span> /**<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span> * Get procedures.<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span> * @return the procedures in a list<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span> */<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span> public List<Procedure<TEnvironment>> getProcedures() {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span> List<Procedure<TEnvironment>> procedureList =<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span> new ArrayList<>(procedures.size() + completed.size());<a name="line.1257"></a>
+<span class="sourceLineNo">1258</span> procedureList.addAll(procedures.values());<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span> // Note: The procedure could show up twice in the list with different state, as<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span> // it could complete after we walk through procedures list and insert into<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span> // procedureList - it is ok, as we will use the information in the Procedure<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span> // to figure it out; to prevent this would increase the complexity of the logic.<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span> completed.values().stream().map(CompletedProcedureRetainer::getProcedure)<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span> .forEach(procedureList::add);<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span> return procedureList;<a name="line.1265"></a>
<span class="sourceLineNo">1266</span> }<a name="line.1266"></a>
<span class="sourceLineNo">1267</span><a name="line.1267"></a>
-<span class="sourceLineNo">1268</span> public boolean unregisterListener(ProcedureExecutorListener listener) {<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span> return this.listeners.remove(listener);<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span> }<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span><a name="line.1271"></a>
-<span class="sourceLineNo">1272</span> private void sendProcedureLoadedNotification(final long procId) {<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span> if (!this.listeners.isEmpty()) {<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span> try {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span> listener.procedureLoaded(procId);<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span> } catch (Throwable e) {<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span> }<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span> }<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span> }<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span> }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span> private void sendProcedureAddedNotification(final long procId) {<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span> if (!this.listeners.isEmpty()) {<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span> try {<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span> listener.procedureAdded(procId);<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span> } catch (Throwable e) {<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span> }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span> }<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span> }<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span> }<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span><a name="line.1295"></a>
-<span class="sourceLineNo">1296</span> private void sendProcedureFinishedNotification(final long procId) {<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span> if (!this.listeners.isEmpty()) {<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span> try {<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span> listener.procedureFinished(procId);<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span> } catch (Throwable e) {<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span> }<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span> }<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span> }<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span> }<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span><a name="line.1307"></a>
-<span class="sourceLineNo">1308</span> // ==========================================================================<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span> // Procedure IDs helpers<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span> // ==========================================================================<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span> private long nextProcId() {<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span> long procId = lastProcId.incrementAndGet();<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span> if (procId < 0) {<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span> while (!lastProcId.compareAndSet(procId, 0)) {<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span> procId = lastProcId.get();<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span> if (procId >= 0) {<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span> break;<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span> }<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span> }<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span> while (procedures.containsKey(procId)) {<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span> procId = lastProcId.incrementAndGet();<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span> }<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span> }<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span> assert procId >= 0 : "Invalid procId " + procId;<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span> return procId;<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span> }<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span><a name="line.1327"></a>
-<span class="sourceLineNo">1328</span> @VisibleForTesting<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span> protected long getLastProcId() {<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span> return lastProcId.get();<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span> }<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span><a name="line.1332"></a>
-<span class="sourceLineNo">1333</span> @VisibleForTesting<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span> public Set<Long> getActiveProcIds() {<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span> return procedures.keySet();<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span> }<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span><a name="line.1337"></a>
-<span class="sourceLineNo">1338</span> Long getRootProcedureId(Procedure<TEnvironment> proc) {<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span> return Procedure.getRootProcedureId(procedures, proc);<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span> }<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span><a name="line.1341"></a>
-<span class="sourceLineNo">1342</span> // ==========================================================================<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span> // Executions<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span> // ==========================================================================<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span> private void executeProcedure(Procedure<TEnvironment> proc) {<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span> if (proc.isFinished()) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span> LOG.debug("{} is already finished, skipping execution", proc);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span> return;<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span> }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span> final Long rootProcId = getRootProcedureId(proc);<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span> if (rootProcId == null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span> // The 'proc' was ready to run but the root procedure was rolledback<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span> LOG.warn("Rollback because parent is done/rolledback proc=" + proc);<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span> executeRollback(proc);<a name="line.1354"></a>
+<span class="sourceLineNo">1268</span> // ==========================================================================<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span> // Listeners helpers<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span> // ==========================================================================<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span> public void registerListener(ProcedureExecutorListener listener) {<a name="line.1271"></a>
+<span class="sourceLineNo">1272</span> this.listeners.add(listener);<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span> }<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span><a name="line.1274"></a>
+<span class="sourceLineNo">1275</span> public boolean unregisterListener(ProcedureExecutorListener listener) {<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span> return this.listeners.remove(listener);<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span> }<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span><a name="line.1278"></a>
+<span class="sourceLineNo">1279</span> private void sendProcedureLoadedNotification(final long procId) {<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span> if (!this.listeners.isEmpty()) {<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1281"></a>
+<span class="sourceLineNo">1282</span> try {<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span> listener.procedureLoaded(procId);<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span> } catch (Throwable e) {<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span> }<a name="line.1286"></a>
+<span class="sourceLineNo">1287</span> }<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span> }<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span> }<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span><a name="line.1290"></a>
+<span class="sourceLineNo">1291</span> private void sendProcedureAddedNotification(final long procId) {<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span> if (!this.listeners.isEmpty()) {<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span> try {<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span> listener.procedureAdded(procId);<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span> } catch (Throwable e) {<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span> }<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span> }<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span> }<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span> }<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span><a name="line.1302"></a>
+<span class="sourceLineNo">1303</span> private void sendProcedureFinishedNotification(final long procId) {<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span> if (!this.listeners.isEmpty()) {<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span> try {<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span> listener.procedureFinished(procId);<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span> } catch (Throwable e) {<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span> }<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span> }<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span> }<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span> }<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span> // ==========================================================================<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span> // Procedure IDs helpers<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span> // ==========================================================================<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span> private long nextProcId() {<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span> long procId = lastProcId.incrementAndGet();<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span> if (procId < 0) {<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span> while (!lastProcId.compareAndSet(procId, 0)) {<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span> procId = lastProcId.get();<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span> if (procId >= 0) {<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span> break;<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span> }<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span> }<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span> while (procedures.containsKey(procId)) {<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span> procId = lastProcId.incrementAndGet();<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span> }<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span> }<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span> assert procId >= 0 : "Invalid procId " + procId;<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span> return procId;<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span> }<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span><a name="line.1334"></a>
+<span class="sourceLineNo">1335</span> @VisibleForTesting<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span> protected long getLastProcId() {<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span> return lastProcId.get();<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span> }<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span><a name="line.1339"></a>
+<span class="sourceLineNo">1340</span> @VisibleForTesting<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span> public Set<Long> getActiveProcIds() {<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span> return procedures.keySet();<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span> }<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span><a name="line.1344"></a>
+<span class="sourceLineNo">1345</span> Long getRootProcedureId(Procedure<TEnvironment> proc) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span> return Procedure.getRootProcedureId(procedures, proc);<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span> }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span> // ==========================================================================<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span> // Executions<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span> // ==========================================================================<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span> private void executeProcedure(Procedure<TEnvironment> proc) {<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span> if (proc.isFinished()) {<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span> LOG.debug("{} is already finished, skipping execution", proc);<a name="line.1354"></a>
<span class="sourceLineNo">1355</span> return;<a name="line.1355"></a>
<span class="sourceLineNo">1356</span> }<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span><a name="line.1357"></a>
-<span class="sourceLineNo">1358</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span> if (procStack == null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span> LOG.warn("RootProcedureState is null for " + proc.getProcId());<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span> return;<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span> }<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span> do {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span> // Try to acquire the execution<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span> if (!procStack.acquire(proc)) {<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span> if (procStack.setRollback()) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span> // we have the 'rollback-lock' we can start rollingback<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span> switch (executeRollback(rootProcId, procStack)) {<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span> case LOCK_ACQUIRED:<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span> break;<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span> case LOCK_YIELD_WAIT:<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span> procStack.unsetRollback();<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span> scheduler.yield(proc);<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span> break;<a name="line.1374"></a>
-<span class="sourceLineNo">1375</span> case LOCK_EVENT_WAIT:<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span> LOG.info("LOCK_EVENT_WAIT rollback..." + proc);<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span> procStack.unsetRollback();<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span> break;<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span> default:<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span> throw new UnsupportedOperationException();<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span> }<a name="line.1381"></a>
-<span class="sourceLineNo">1382</span> } else {<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span> // if we can't rollback means that some child is still running.<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span> // the rollback will be executed after all the children are done.<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span> // If the procedure was never executed, remove and mark it as rolledback.<a name="line.1385"></a>
-<span class="sourceLineNo">1386</span> if (!proc.wasExecuted()) {<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span> switch (executeRollback(proc)) {<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span> case LOCK_ACQUIRED:<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span> break;<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span> case LOCK_YIELD_WAIT:<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span> scheduler.yield(proc);<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span> break;<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span> case LOCK_EVENT_WAIT:<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span> LOG.info("LOCK_EVENT_WAIT can't rollback child running?..." + proc);<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span> break;<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span> default:<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span> throw new UnsupportedOperationException();<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span> }<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span> }<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span> }<a name="line.1400"></a>
-<span class="sourceLineNo">1401</span> break;<a name="line.1401"></a>
-<span class="sourceLineNo">1402</span> }<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span><a name="line.1403"></a>
-<span class="sourceLineNo">1404</span> // Execute the procedure<a name="line.1404"></a>
-<span class="sourceLineNo">1405</span> assert proc.getState() == ProcedureState.RUNNABLE : proc;<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span> // Note that lock is NOT about concurrency but rather about ensuring<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span> // ownership of a procedure of an entity such as a region or table<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span> LockState lockState = acquireLock(proc);<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span> switch (lockState) {<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span> case LOCK_ACQUIRED:<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span> execProcedure(procStack, proc);<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span> break;<a name="line.1412"></a>
-<span class="sourceLineNo">1413</span> case LOCK_YIELD_WAIT:<a name="line.1413"></a>
-<span class="sourceLineNo">1414</span> LOG.info(lockState + " " + proc);<a name="line.1414"></a>
-<span class="sourceLineNo">1415</span> scheduler.yield(proc);<a name="line.1415"></a>
-<span class="sourceLineNo">1416</span> break;<a name="line.1416"></a>
-<span class="sourceLineNo">1417</span> case LOCK_EVENT_WAIT:<a name="line.1417"></a>
-<span class="sourceLineNo">1418</span> // Someone will wake us up when the lock is available<a name="line.1418"></a>
-<span class="sourceLineNo">1419</span> LOG.debug(lockState + " " + proc);<a name="line.1419"></a>
-<span class="sourceLineNo">1420</span> break;<a name="line.1420"></a>
-<span class="sourceLineNo">1421</span> default:<a name="line.1421"></a>
-<span class="sourceLineNo">1422</span> throw new UnsupportedOperationException();<a name="line.1422"></a>
-<span class="sourceLineNo">1423</span> }<a name="line.1423"></a>
-<span class="sourceLineNo">1424</span> procStack.release(proc);<a name="line.1424"></a>
-<span class="sourceLineNo">1425</span><a name="line.1425"></a>
-<span class="sourceLineNo">1426</span> if (proc.isSuccess()) {<a name="line.1426"></a>
-<span class="sourceLineNo">1427</span> // update metrics on finishing the procedure<a name="line.1427"></a>
-<span class="sourceLineNo">1428</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);<a name="line.1428"></a>
-<span class="sourceLineNo">1429</span> LOG.info("Finished " + proc + " in " + StringUtils.humanTimeDiff(proc.elapsedTime()));<a name="line.1429"></a>
-<span class="sourceLineNo">1430</span> // Finalize the procedure state<a name="line.1430"></a>
-<span class="sourceLineNo">1431</span> if (proc.getProcId() == rootProcId) {<a name="line.1431"></a>
-<span class="sourceLineNo">1432</span> procedureFinished(proc);<a name="line.1432"></a>
-<span class="sourceLineNo">1433</span> } else {<a name="line.1433"></a>
-<span class="sourceLineNo">1434</span> execCompletionCleanup(proc);<a name="line.1434"></a>
-<span class="sourceLineNo">1435</span> }<a name="line.1435"></a>
-<span class="sourceLineNo">1436</span> break;<a name="line.1436"></a>
-<span class="sourceLineNo">1437</span> }<a name="line.1437"></a>
-<span class="sourceLineNo">1438</span> } while (procStack.isFailed());<a name="line.1438"></a>
-<span class="sourceLineNo">1439</span> }<a name="line.1439"></a>
-<span class="sourceLineNo">1440</span><a name="line.1440"></a>
-<span class="sourceLineNo">1441</span> private LockState acquireLock(Procedure<TEnvironment> proc) {<a name="line.1441"></a>
-<span class="sourceLineNo">1442</span> TEnvironment env = getEnvironment();<a name="line.1442"></a>
-<span class="sourceLineNo">1443</span> // if holdLock is true, then maybe we already have the lock, so just return LOCK_ACQUIRED if<a name="line.1443"></a>
-<span class="sourceLineNo">1444</span> // hasLock is true.<a name="line.1444"></a>
-<span class="sourceLineNo">1445</span> if (proc.hasLock()) {<a name="line.1445"></a>
-<span class="sourceLineNo">1446</span> return LockState.LOCK_ACQUIRED;<a name="line.1446"></a>
-<span class="sourceLineNo">1447</span> }<a name="line.1447"></a>
-<span class="sourceLineNo">1448</span> return proc.doAcquireLock(env, store);<a name="line.1448"></a>
-<span class="sourceLineNo">1449</span> }<a name="line.1449"></a>
-<span class="sourceLineNo">1450</span><a name="line.1450"></a>
-<span class="sourceLineNo">1451</span> private void releaseLock(Procedure<TEnvironment> proc, boolean force) {<a name="line.1451"></a>
-<span class="sourceLineNo">1452</span> TEnvironment env = getEnvironment();<a name="line.1452"></a>
-<span class="sourceLineNo">1453</span> // For how the framework works, we know that we will always have the lock<a name="line.1453"></a>
-<span class="sourceLineNo">1454</span> // when we call releaseLock(), so we can avoid calling proc.hasLock()<a name="line.1454"></a>
-<span class="sourceLineNo">1455</span> if (force || !proc.holdLock(env) || proc.isFinished()) {<a name="line.1455"></a>
-<span class="sourceLineNo">1456</span> proc.doReleaseLock(env, store);<a name="line.1456"></a>
-<span class="sourceLineNo">1457</span> }<a name="line.1457"></a>
-<span class="sourceLineNo">1458</span> }<a name="line.1458"></a>
-<span class="sourceLineNo">1459</span><a name="line.1459"></a>
-<span class="sourceLineNo">1460</span> /**<a name="line.1460"></a>
-<span class="sourceLineNo">1461</span> * Execute the rollback of the full procedure stack. Once the procedure is rolledback, the<a name="line.1461"></a>
-<span class="sourceLineNo">1462</span> * root-procedure will be visible as finished to user, and the result will be the fatal exception.<a name="line.1462"></a>
-<span class="sourceLineNo">1463</span> */<a name="line.1463"></a>
-<span class="sourceLineNo">1464</span> private LockState executeRollback(long rootProcId, RootProcedureState<TEnvironment> procStack) {<a name="line.1464"></a>
-<span class="sourceLineNo">1465</span> Procedure<TEnvironment> rootProc = procedures.get(rootProcId);<a name="line.1465"></a>
-<span class="sourceLineNo">1466</span> RemoteProcedureException exception = rootProc.getException();<a name="line.1466"></a>
-<span class="sourceLineNo">1467</span> // TODO: This needs doc. The root proc doesn't have an exception. Maybe we are<a name="line.1467"></a>
-<span class="sourceLineNo">1468</span> // rolling back because the subprocedure does. Clarify.<a name="line.1468"></a>
-<span class="sourceLineNo">1469</span> if (exception == null) {<a name="line.1469"></a>
-<span class="sourceLineNo">1470</span> exception = procStack.getException();<a name="line.1470"></a>
-<span class="sourceLineNo">1471</span> rootProc.setFailure(exception);<a name="line.1471"></a>
-<span class="sourceLineNo">1472</span> store.update(rootProc);<a name="line.1472"></a>
-<span class="sourceLineNo">1473</span> }<a name="line.1473"></a>
-<span class="sourceLineNo">1474</span><a name="line.1474"></a>
-<span class="sourceLineNo">1475</span> List<Procedure<TEnvironment>> subprocStack = procStack.getSubproceduresStack();<a name="line.1475"></a>
-<span class="sourceLineNo">1476</span> assert subprocStack != null : "Called rollback with no steps executed rootProc=" + rootProc;<a name="line.1476"></a>
-<span class="sourceLineNo">1477</span><a name="line.1477"></a>
-<span class="sourceLineNo">1478</span> int stackTail = subprocStack.size();<a name="line.1478"></a>
-<span class="sourceLineNo">1479</span> while (stackTail-- > 0) {<a name="line.1479"></a>
-<span class="sourceLineNo">1480</span> Procedure<TEnvironment> proc = subprocStack.get(stackTail);<a name="line.1480"></a>
-<span class="sourceLineNo">1481</span> IdLock.Entry lockEntry = null;<a name="line.1481"></a>
-<span class="sourceLineNo">1482</span> // Hold the execution lock if it is not held by us. The IdLock is not reentrant so we need<a name="line.1482"></a>
-<span class="sourceLineNo">1483</span> // this check, as the worker will hold the lock before executing a procedure. This is the only<a name="line.1483"></a>
-<span class="sourceLineNo">1484</span> // place where we may hold two procedure execution locks, and there is a fence in the<a name="line.1484"></a>
-<span class="sourceLineNo">1485</span> // RootProcedureState where we can make sure that only one worker can execute the rollback of<a name="line.1485"></a>
-<span class="sourceLineNo">1486</span> // a RootProcedureState, so there is no dead lock problem. And the lock here is necessary to<a name="line.1486"></a>
-<span class="sourceLineNo">1487</span> // prevent race between us and the force update thread.<a name="line.1487"></a>
-<span class="sourceLineNo">1488</span> if (!procExecutionLock.isHeldByCurrentThread(proc.getProcId())) {<a name="line.1488"></a>
-<span class="sourceLineNo">1489</span> try {<a name="line.1489"></a>
-<span class="sourceLineNo">1490</span> lockEntry = procExecutionLock.getLockEntry(proc.getProcId());<a name="line.1490"></a>
-<span class="sourceLineNo">1491</span> } catch (IOException e) {<a name="line.1491"></a>
-<span class="sourceLineNo">1492</span> // can only happen if interrupted, so not a big deal to propagate it<a name="line.1492"></a>
-<span class="sourceLineNo">1493</span> throw new UncheckedIOException(e);<a name="line.1493"></a>
-<span class="sourceLineNo">1494</span> }<a name="line.1494"></a>
-<span class="sourceLineNo">1495</span> }<a name="line.1495"></a>
-<span class="sourceLineNo">1496</span> try {<a name="line.1496"></a>
-<span class="sourceLineNo">1497</span> // For the sub procedures which are successfully finished, we do not rollback them.<a name="line.1497"></a>
-<span class="sourceLineNo">1498</span> // Typically, if we want to rollback a procedure, we first need to rollback it, and then<a name="line.1498"></a>
-<span class="sourceLineNo">1499</span> // recursively rollback its ancestors. The state changes which are done by sub procedures<a name="line.1499"></a>
-<span class="sourceLineNo">1500</span> // should be handled by parent procedures when rolling back. For example, when rolling back<a name="line.1500"></a>
-<span class="sourceLineNo">1501</span> // a MergeTableProcedure, we will schedule new procedures to bring the offline regions<a name="line.1501"></a>
-<span class="sourceLineNo">1502</span> // online, instead of rolling back the original procedures which offlined the regions(in<a name="line.1502"></a>
-<span class="sourceLineNo">1503</span> // fact these procedures can not be rolled back...).<a name="line.1503"></a>
-<span class="sourceLineNo">1504</span> if (proc.isSuccess()) {<a name="line.1504"></a>
-<span class="sourceLineNo">1505</span> // Just do the cleanup work, without actually executing the rollback<a name="line.1505"></a>
-<span class="sourceLineNo">1506</span> subprocStack.remove(stackTail);<a name="line.1506"></a>
-<span class="sourceLineNo">1507</span> cleanupAfterRollbackOneStep(proc);<a name="line.1507"></a>
-<span class="sourceLineNo">1508</span> continue;<a name="line.1508"></a>
-<span class="sourceLineNo">1509</span> }<a name="line.1509"></a>
-<span class="sourceLineNo">1510</span> LockState lockState = acquireLock(proc);<a name="line.1510"></a>
-<span class="sourceLineNo">1511</span> if (lockState != LockState.LOCK_ACQUIRED) {<a name="line.1511"></a>
-<span class="sourceLineNo">1512</span> // can't take a lock on the procedure, add the root-proc back on the<a name="line.1512"></a>
-<span class="sourceLineNo">1513</span> // queue waiting for the lock availability<a name="line.1513"></a>
-<span class="sourceLineNo">1514</span> return lockState;<a name="line.1514"></a>
-<span class="sourceLineNo">1515</span> }<a name="line.1515"></a>
-<span class="sourceLineNo">1516</span><a name="line.1516"></a>
-<span class="sourceLineNo">1517</span> lockState = executeRollback(proc);<a name="line.1517"></a>
-<span class="sourceLineNo">1518</span> releaseLock(proc, false);<a name="line.1518"></a>
-<span class="sourceLineNo">1519</span> boolean abortRollback = lockState != LockState.LOCK_ACQUIRED;<a name="line.1519"></a>
-<span class="sourceLineNo">1520</span> abortRollback |= !isRunning() || !store.isRunning();<a name="line.1520"></a>
-<span class="sourceLineNo">1521</span><a name="line.1521"></a>
-<span class="sourceLineNo">1522</span> // allows to kill the executor before something is stored to the wal.<a name="line.1522"></a>
-<span class="sourceLineNo">1523</span> // useful to test the procedure recovery.<a name="line.1523"></a>
-<span class="sourceLineNo">1524</span> if (abortRollback) {<a name="line.1524"></a>
-<span class="sourceLineNo">1525</span> return lockState;<a name="line.1525"></a>
-<span class="sourceLineNo">1526</span> }<a name="line.1526"></a>
-<span class="sourceLineNo">1527</span><a name="line.1527"></a>
-<span class="sourceLineNo">1528</span> subprocStack.remove(stackTail);<a name="line.1528"></a>
-<span class="sourceLineNo">1529</span><a name="line.1529"></a>
-<span class="sourceLineNo">1530</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1530"></a>
-<span class="sourceLineNo">1531</span> // if the proc is already finished, do not yield<a name="line.1531"></a>
-<span class="sourceLineNo">1532</span> if (!proc.isFinished() && proc.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1532"></a>
-<span class="sourceLineNo">1533</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1533"></a>
-<span class="sourceLineNo">1534</span> }<a name="line.1534"></a>
-<span class="sourceLineNo">1535</span><a name="line.1535"></a>
-<span class="sourceLineNo">1536</span> if (proc != rootProc) {<a name="line.1536"></a>
-<span class="sourceLineNo">1537</span> execCompletionCleanup(proc);<a name="line.1537"></a>
-<span class="sourceLineNo">1538</span> }<a name="line.1538"></a>
-<span class="sourceLineNo">1539</span> } finally {<a name="line.1539"></a>
-<span class="sourceLineNo">1540</span> if (lockEntry != null) {<a name="line.1540"></a>
-<span class="sourceLineNo">1541</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1541"></a>
-<span class="sourceLineNo">1542</span> }<a name="line.1542"></a>
-<span class="sourceLineNo">1543</span> }<a name="line.1543"></a>
-<span class="sourceLineNo">1544</span> }<a name="line.1544"></a>
-<span class="sourceLineNo">1545</span><a name="line.1545"></a>
-<span class="sourceLineNo">1546</span> // Finalize the procedure state<a name="line.1546"></a>
-<span class="sourceLineNo">1547</span> LOG.info("Rolled back {} exec-time={}", rootProc,<a name="line.1547"></a>
-<span class="sourceLineNo">1548</span> StringUtils.humanTimeDiff(rootProc.elapsedTime()));<a name="line.1548"></a>
-<span class="sourceLineNo">1549</span> procedureFinished(rootProc);<a name="line.1549"></a>
-<span class="sourceLineNo">1550</span> return LockState.LOCK_ACQUIRED;<a name="line.1550"></a>
-<span class="sourceLineNo">1551</span> }<a name="line.1551"></a>
+<span class="sourceLineNo">1357</span> final Long rootProcId = getRootProcedureId(proc);<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span> if (rootProcId == null) {<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span> // The 'proc' was ready to run but the root procedure was rolledback<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span> LOG.warn("Rollback because parent is done/rolledback proc=" + proc);<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span> executeRollback(proc);<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span> return;<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span> }<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span><a name="line.1364"></a>
+<span class="sourceLineNo">1365</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span> if (procStack == null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span> LOG.warn("RootProcedureState is null for " + proc.getProcId());<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span> return;<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span> }<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span> do {<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span> // Try to acquire the execution<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span> if (!procStack.acquire(proc)) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span> if (procStack.setRollback()) {<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span> // we have the 'rollback-lock' we can start rollingback<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span> switch (executeRollback(rootProcId, procStack)) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span> case LOCK_ACQUIRED:<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span> break;<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span> case LOCK_YIELD_WAIT:<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span> procStack.unsetRollback();<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span> scheduler.yield(proc);<a name="line.1380"></a>
+<span class="sourceLineNo">1381</span> break;<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span> case LOCK_EVENT_WAIT:<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span> LOG.info("LOCK_EVENT_WAIT rollback..." + proc);<a name="line.1383"></a>
+<span class="sourceLineNo">1384</span> procStack.unsetRollback();<a name="line.1384"></a>
+<span class="sourceLineNo">1385</span> break;<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span> default:<a name="line.1386"></a>
+<span class="sourceLineNo">1387</span> throw new UnsupportedOperationException();<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span> }<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span> } else {<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span> // if we can't rollback means that some child is still running.<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span> // the rollback will be executed after all the children are done.<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span> // If the procedure was never executed, remove and mark it as rolledback.<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span> if (!proc.wasExecuted()) {<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span> switch (executeRollback(proc)) {<a name="line.1394"></a>
+<span class="sourceLineNo">1395</span> case LOCK_ACQUIRED:<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span> break;<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span> case LOCK_YIELD_WAIT:<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span> scheduler.yield(proc);<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span> break;<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span> case LOCK_EVENT_WAIT:<a name="line.1400"></a>
+<span class="sourceLineNo">1401</span> LOG.info("LOCK_EVENT_WAIT can't rollback child running?..." + proc);<a name="line.1401"></a>
+<span class="sourceLineNo">1402</span> break;<a name="line.1402"></a>
+<span class="sourceLineNo">1403</span> default:<a name="line.1403"></a>
+<span class="sourceLineNo">1404</span> throw new UnsupportedOperationException();<a name="line.1404"></a>
+<span class="sourceLineNo">1405</span> }<a name="line.1405"></a>
+<span class="sourceLineNo">1406</span> }<a name="line.1406"></a>
+<span class="sourceLineNo">1407</span> }<a name="line.1407"></a>
+<span class="sourceLineNo">1408</span> break;<a name="line.1408"></a>
+<span class="sourceLineNo">1409</span> }<a name="line.1409"></a>
+<span class="sourceLineNo">1410</span><a name="line.1410"></a>
+<span class="sourceLineNo">1411</span> // Execute the procedure<a name="line.1411"></a>
+<span class="sourceLineNo">1412</span> assert proc.getState() == ProcedureState.RUNNABLE : proc;<a name="line.1412"></a>
+<span class="sourceLineNo">1413</span> // Note that lock is NOT about concurrency but rather about ensuring<a name="line.1413"></a>
+<span class="sourceLineNo">1414</span> // ownership of a procedure of an entity such as a region or table<a name="line.1414"></a>
+<span class="sourceLineNo">1415</span> LockState lockState = acquireLock(proc);<a name="line.1415"></a>
+<span class="sourceLineNo">1416</span> switch (lockState) {<a name="line.1416"></a>
+<span class="sourceLineNo">1417</span> case LOCK_ACQUIRED:<a name="line.1417"></a>
+<span class="sourceLineNo">1418</span> execProcedure(procStack, proc);<a name="line.1418"></a>
+<span class="sourceLineNo">1419</span> break;<a name="line.1419"></a>
+<span class="sourceLineNo">1420</span> case LOCK_YIELD_WAIT:<a name="line.1420"></a>
+<span class="sourceLineNo">1421</span> LOG.info(lockState + " " + proc);<a name="line.1421"></a>
+<span class="sourceLineNo">1422</span> scheduler.yield(proc);<a name="line.1422"></a>
+<span class="sourceLineNo">1423</span> break;<a name="line.1423"></a>
+<span class="sourceLineNo">1424</span> case LOCK_EVENT_WAIT:<a name="line.1424"></a>
+<span class="sourceLineNo">1425</span> // Someone will wake us up when the lock is available<a name="line.1425"></a>
+<span class="sourceLineNo">1426</span> LOG.debug(lockState + " " + proc);<a name="line.1426"></a>
+<span class="sourceLineNo">1427</span> break;<a name="line.1427"></a>
+<span class="sourceLineNo">1428</span> default:<a name="line.1428"></a>
+<span class="sourceLineNo">1429</span> throw new UnsupportedOperationException();<a name="line.1429"></a>
+<span class="sourceLineNo">1430</span> }<a name="line.1430"></a>
+<span class="sourceLineNo">1431</span> procStack.release(proc);<a name="line.1431"></a>
+<span class="sourceLineNo">1432</span><a name="line.1432"></a>
+<span class="sourceLineNo">1433</span> if (proc.isSuccess()) {<a name="line.1433"></a>
+<span class="sourceLineNo">1434</span> // update metrics on finishing the procedure<a name="line.1434"></a>
+<span class="sourceLineNo">1435</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);<a name="line.1435"></a>
+<span class="sourceLineNo">1436</span> LOG.info("Finished " + proc + " in " + StringUtils.humanTimeDiff(proc.elapsedTime()));<a name="line.1436"></a>
+<span class="sourceLineNo">1437</span> // Finalize the procedure state<a name="line.1437"></a>
+<span class="sourceLineNo">1438</span> if (proc.getProcId() == rootProcId) {<a name="line.1438"></a>
+<span class="sourceLineNo">1439</span> procedureFinished(proc);<a name="line.1439"></a>
+<span class="sourceLineNo">1440</span> } else {<a name="line.1440"></a>
+<span class="sourceLineNo">1441</span> execCompletionCleanup(proc);<a name="line.1441"></a>
+<span class="sourceLineNo">1442</span> }<a name="line.1442"></a>
+<span class="sourceLineNo">1443</span> break;<a name="line.1443"></a>
+<span class="sourceLineNo">1444</span> }<a name="line.1444"></a>
+<span class="sourceLineNo">1445</span> } while (procStack.isFailed());<a name="line.1445"></a>
+<span class="sourceLineNo">1446</span> }<a name="line.1446"></a>
+<span class="sourceLineNo">1447</span><a name="line.1447"></a>
+<span class="sourceLineNo">1448</span> private LockState acquireLock(Procedure<TEnvironment> proc) {<a name="line.1448"></a>
+<span class="sourceLineNo">1449</span> TEnvironment env = getEnvironment();<a name="line.1449"></a>
+<span class="sourceLineNo">1450</span> // if holdLock is true, then maybe we already have the lock, so just return LOCK_ACQUIRED if<a name="line.1450"></a>
+<span class="sourceLineNo">1451</span> // hasLock is true.<a name="line.1451"></a>
+<span class="sourceLineNo">1452</span> if (proc.hasLock()) {<a name="line.1452"></a>
+<span class="sourceLineNo">1453</span> return LockState.LOCK_ACQUIRED;<a name="line.1453"></a>
+<span class="sourceLineNo">1454</span> }<a name="line.1454"></a>
+<span class="sourceLineNo">1455</span> return proc.doAcquireLock(env, store);<a name="line.1455"></a>
+<span class="sourceLineNo">1456</span> }<a name="line.1456"></a>
+<span class="sourceLineNo">1457</span><a name="line.1457"></a>
+<span class="sourceLineNo">1458</span> private void releaseLock(Procedure<TEnvironment> proc, boolean force) {<a name="line.1458"></a>
+<span class="sourceLineNo">1459</span> TEnvironment env = getEnvironment();<a name="line.1459"></a>
+<span class="sourceLineNo">1460</span> // For how the framework works, we know that we will always have the lock<a name="line.1460"></a>
+<span class="sourceLineNo">1461</span> // when we call releaseLock(), so we can avoid calling proc.hasLock()<a name="line.1461"></a>
+<span class="sourceLineNo">1462</span> if (force || !proc.holdLock(env) || proc.isFinished()) {<a name="line.1462"></a>
+<span class="sourceLineNo">1463</span> proc.doReleaseLock(env, store);<a name="line.1463"></a>
+<span class="sourceLineNo">1464</span> }<a name="line.1464"></a>
+<span class="sourceLineNo">1465</span> }<a name="line.1465"></a>
+<span class="sourceLineNo">1466</span><a name="line.1466"></a>
+<span class="sourceLineNo">1467</span> /**<a name="line.1467"></a>
+<span class="sourceLineNo">1468</span> * Execute the rollback of the full procedure stack. Once the procedure is rolledback, the<a name="line.1468"></a>
+<span class="sourceLineNo">1469</span> * root-procedure will be visible as finished to user, and the result will be the fatal exception.<a name="line.1469"></a>
+<span class="sourceLineNo">1470</span> */<a name="line.1470"></a>
+<span class="sourceLineNo">1471</span> private LockState executeRollback(long rootProcId, RootProcedureState<TEnvironment> procStack) {<a name="line.1471"></a>
+<span class="sourceLineNo">1472</span> Procedure<TEnvironment> rootProc = procedures.get(rootProcId);<a name="line.1472"></a>
+<span class="sourceLineNo">1473</span> RemoteProcedureException exception = rootProc.getException();<a name="line.1473"></a>
+<span class="sourceLineNo">1474</span> // TODO: This needs doc. The root proc doesn't have an exception. Maybe we are<a name="line.1474"></a>
+<span class="sourceLineNo">1475</span> // rolling back because the subprocedure does. Clarify.<a name="line.1475"></a>
+<span class="sourceLineNo">1476</span> if (exception == null) {<a name="line.1476"></a>
+<span class="sourceLineNo">1477</span> exception = procStack.getException();<a name="line.1477"></a>
+<span class="sourceLineNo">1478</span> rootProc.setFailure(exception);<a name="line.1478"></a>
+<span class="sourceLineNo">1479</span> store.update(rootProc);<a name="line.1479"></a>
+<span class="sourceLineNo">1480</span> }<a name="line.1480"></a>
+<span class="sourceLineNo">1481</span><a name="line.1481"></a>
+<span class="sourceLineNo">1482</span> List<Procedure<TEnvironment>> subprocStack = procStack.getSubproceduresStack();<a name="line.1482"></a>
+<span class="sourceLineNo">1483</span> assert subprocStack != null : "Called rollback with no steps executed rootProc=" + rootProc;<a name="line.1483"></a>
+<span class="sourceLineNo">1484</span><a name="line.1484"></a>
+<span class="sourceLineNo">1485</span> int stackTail = subprocStack.size();<a name="line.1485"></a>
+<span class="sourceLineNo">1486</span> while (stackTail-- > 0) {<a name="line.1486"></a>
+<span class="sourceLineNo">1487</span> Procedure<TEnvironment> proc = subprocStack.get(stackTail);<a name="line.1487"></a>
+<span class="sourceLineNo">1488</span> IdLock.Entry lockEntry = null;<a name="line.1488"></a>
+<span class="sourceLineNo">1489</span> // Hold the execution lock if it is not held by us. The IdLock is not reentrant so we need<a name="line.1489"></a>
+<span class="sourceLineNo">1490</span> // this check, as the worker will hold the lock before executing a procedure. This is the only<a name="line.1490"></a>
+<span class="sourceLineNo">1491</span> // place where we may hold two procedure execution locks, and there is a fence in the<a name="line.1491"></a>
+<span class="sourceLineNo">1492</span> // RootProcedureState where we can make sure that only one worker can execute the rollback of<a name="line.1492"></a>
+<span class="sourceLineNo">1493</span> // a RootProcedureState, so there is no dead lock problem. And the lock here is necessary to<a name="line.1493"></a>
+<span class="sourceLineNo">1494</span> // prevent race between us and the force update thread.<a name="line.1494"></a>
+<span class="sourceLineNo">1495</span> if (!procExecutionLock.isHeldByCurrentThread(proc.getProcId())) {<a name="line.1495"></a>
+<span class="sourceLineNo">1496</span> try {<a name="line.1496"></a>
+<span class="sourceLineNo">1497</span> lockEntry = procExecutionLock.getLockEntry(proc.getProcId());<a name="line.1497"></a>
+<span class="sourceLineNo">1498</span> } catch (IOException e) {<a name="line.1498"></a>
+<span class="sourceLineNo">1499</span> // can only happen if interrupted, so not a big deal to propagate it<a name="line.1499"></a>
+<span class="sourceLineNo">1500</span> throw new UncheckedIOException(e);<a name="line.1500"></a>
+<span class="sourceLineNo">1501</span> }<a name="line.1501"></a>
+<span class="sourceLineNo">1502</span> }<a name="line.1502"></a>
+<span class="sourceLineNo">1503</span> try {<a name="line.1503"></a>
+<span class="sourceLineNo">1504</span> // For the sub procedures which are successfully finished, we do not rollback them.<a name="line.1504"></a>
+<span class="sourceLineNo">1505</span> // Typically, if we want to rollback a procedure, we first need to rollback it, and then<a name="line.1505"></a>
+<span class="sourceLineNo">1506</span> // recursively rollback its ancestors. The state changes which are done by sub procedures<a name="line.1506"></a>
+<span class="sourceLineNo">1507</span> // should be handled by parent procedures when rolling back. For example, when rolling back<a name="line.1507"></a>
+<span class="sourceLineNo">1508</span> // a MergeTableProcedure, we will schedule new procedures to bring the offline regions<a name="line.1508"></a>
+<span class="sourceLineNo">1509</span> // online, instead of rolling back the original procedures which offlined the regions(in<a name="line.1509"></a>
+<span class="sourceLineNo">1510</span> // fact these procedures can not be rolled back...).<a name="line.1510"></a>
+<span class="sourceLineNo">1511</span> if (proc.isSuccess()) {<a name="line.1511"></a>
+<span class="sourceLineNo">1512</span> // Just do the cleanup work, without actually executing the rollback<a name="line.1512"></a>
+<span class="sourceLineNo">1513</span> subprocStack.remove(stackTail);<a name="line.1513"></a>
+<span class="sourceLineNo">1514</span> cleanupAfterRollbackOneStep(proc);<a name="line.1514"></a>
+<span class="sourceLineNo">1515</span> continue;<a name="line.1515"></a>
+<span class="sourceLineNo">1516</span> }<a name="line.1516"></a>
+<span class="sourceLineNo">1517</span> LockState lockState = acquireLock(proc);<a name="line.1517"></a>
+<span class="sourceLineNo">1518</span> if (lockState != LockState.LOCK_ACQUIRED) {<a name="line.1518"></a>
+<span class="sourceLineNo">1519</span> // can't take a lock on the procedure, add the root-proc back on the<a name="line.1519"></a>
+<span class="sourceLineNo">1520</span> // queue waiting for the lock availability<a name="line.1520"></a>
+<span class="sourceLineNo">1521</span> return lockState;<a name="line.1521"></a>
+<span class="sourceLineNo">1522</span> }<a name="line.1522"></a>
+<span class="sourceLineNo">1523</span><a name="line.1523"></a>
+<span class="sourceLineNo">1524</span> lockState = executeRollback(proc);<a name="line.1524"></a>
+<span class="sourceLineNo">1525</span> releaseLock(proc, false);<a name="line.1525"></a>
+<span class="sourceLineNo">1526</span> boolean abortRollback = lockState != LockState.LOCK_ACQUIRED;<a name="line.1526"></a>
+<span class="sourceLineNo">1527</span> abortRollback |= !isRunning() || !store.isRunning();<a name="line.1527"></a>
+<span class="sourceLineNo">1528</span><a name="line.1528"></a>
+<span class="sourceLineNo">1529</span> // allows to kill the executor before something is stored to the wal.<a name="line.1529"></a>
+<span class="sourceLineNo">1530</span> // useful to test the procedure recovery.<a name="line.1530"></a>
+<span class="sourceLineNo">1531</span> if (abortRollback) {<a name="line.1531"></a>
+<span class="sourceLineNo">1532</span> return lockState;<a name="line.1532"></a>
+<span class="sourceLineNo">1533</span> }<a name="line.1533"></a>
+<span class="sourceLineNo">1534</span><a name="line.1534"></a>
+<span class="sourceLineNo">1535</span> subprocStack.remove(stackTail);<a name="line.1535"></a>
+<span class="sourceLineNo">1536</span><a name="line.1536"></a>
+<span class="sourceLineNo">1537</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1537"></a>
+<span class="sourceLineNo">1538</span> // if the proc is already finished, do not yield<a name="line.1538"></a>
+<span class="sourceLineNo">1539</span> if (!proc.isFinished() && proc.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1539"></a>
+<span class="sourceLineNo">1540</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1540"></a>
+<span class="sourceLineNo">1541</span> }<a name="line.1541"></a>
+<span class="sourceLineNo">1542</span><a name="line.1542"></a>
+<span class="sourceLineNo">1543</span> if (proc != rootProc) {<a name="line.1543"></a>
+<span class="sourceLineNo">1544</span> execCompletionCleanup(proc);<a name="line.1544"></a>
+<span class="sourceLineNo">1545</span> }<a name="line.1545"></a>
+<span class="sourceLineNo">1546</span> } finally {<a name="line.1546"></a>
+<span class="sourceLineNo">1547</span> if (lockEntry != null) {<a name="line.1547"></a>
+<span class="sourceLineNo">1548</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1548"></a>
+<span class="sourceLineNo">1549</span> }<a name="line.1549"></a>
+<span class="sourceLineNo">1550</span> }<a name="line.1550"></a>
+<span class="sourceLineNo">1551</span> }<a name="line.1551"></a>
<span class="sourceLineNo">1552</span><a name="line.1552"></a>
-<span class="sourceLineNo">1553</span> private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) {<a name="line.1553"></a>
-<span class="sourceLineNo">1554</span> if (proc.removeStackIndex()) {<a name="line.1554"></a>
-<span class="sourceLineNo">1555</span> if (!proc.isSuccess()) {<a name="line.1555"></a>
-<span class="sourceLineNo">1556</span> proc.setState(ProcedureState.ROLLEDBACK);<a name="line.1556"></a>
-<span class="sourceLineNo">1557</span> }<a name="line.1557"></a>
-<span class="sourceLineNo">1558</span><a name="line.1558"></a>
-<span class="sourceLineNo">1559</span> // update metrics on finishing the procedure (fail)<a name="line.1559"></a>
-<span class="sourceLineNo">1560</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false);<a name="line.1560"></a>
-<span class="sourceLineNo">1561</span><a name="line.1561"></a>
-<span class="sourceLineNo">1562</span> if (proc.hasParent()) {<a name="line.1562"></a>
-<span class="sourceLineNo">1563</span> store.delete(proc.getProcId());<a name="line.1563"></a>
-<span class="sourceLineNo">1564</span> procedures.remove(proc.getProcId());<a name="line.1564"></a>
-<span class="sourceLineNo">1565</span> } else {<a name="line.1565"></a>
-<span class="sourceLineNo">1566</span> final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds();<a name="line.1566"></a>
-<span class="sourceLineNo">1567</span> if (childProcIds != null) {<a name="line.1567"></a>
-<span class="sourceLineNo">1568</span> store.delete(proc, childProcIds);<a name="line.1568"></a>
-<span class="sourceLineNo">1569</span> } else {<a name="line.1569"></a>
-<span class="sourceLineNo">1570</span> store.update(proc);<a name="line.1570"></a>
-<span class="sourceLineNo">1571</span> }<a name="line.1571"></a>
-<span class="sourceLineNo">1572</span> }<a name="line.1572"></a>
-<span class="sourceLineNo">1573</span> } else {<a name="line.1573"></a>
-<span class="sourceLineNo">1574</span> store.update(proc);<a name="line.1574"></a>
-<span class="sourceLineNo">1575</span> }<a name="line.1575"></a>
-<span class="sourceLineNo">1576</span> }<a name="line.1576"></a>
-<span class="sourceLineNo">1577</span><a name="line.1577"></a>
-<span class="sourceLineNo">1578</span> /**<a name="line.1578"></a>
-<span class="sourceLineNo">1579</span> * Execute the rollback of the procedure step.<a name="line.1579"></a>
-<span class="sourceLineNo">1580</span> * It updates the store with the new state (stack index)<a name="line.1580"></a>
-<span class="sourceLineNo">1581</span> * or will remove completly the procedure in case it is a child.<a name="line.1581"></a>
-<span class="sourceLineNo">1582</span> */<a name="line.1582"></a>
-<span class="sourceLineNo">1583</span> private LockState executeRollback(Procedure<TEnvironment> proc) {<a name="line.1583"></a>
-<span class="sourceLineNo">1584</span> try {<a name="line.1584"></a>
-<span class="sourceLineNo">1585</span> proc.doRollback(getEnvironment());<a name="line.1585"></a>
-<span class="sourceLineNo">1586</span> } catch (IOException e) {<a name="line.1586"></a>
-<span class="sourceLineNo">1587</span> LOG.debug("Roll back attempt failed for {}", proc, e);<a name="line.1587"></a>
-<span class="sourceLineNo">1588</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1588"></a>
-<span class="sourceLineNo">1589</span> } catch (InterruptedException e) {<a name="line.1589"></a>
-<span class="sourceLineNo">1590</span> handleInterruptedException(proc, e);<a name="line.1590"></a>
-<span class="sourceLineNo">1591</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1591"></a>
-<span class="sourceLineNo">1592</span> } catch (Throwable e) {<a name="line.1592"></a>
-<span class="sourceLineNo">1593</span> // Catch NullPointerExceptions or similar errors...<a name="line.1593"></a>
-<span class="sourceLineNo">1594</span> LOG.error(HBaseMarkers.FATAL, "CODE-BUG: Uncaught runtime exception for " + proc, e);<a name="line.1594"></a>
-<span class="sourceLineNo">1595</span> }<a name="line.1595"></a>
-<span class="sourceLineNo">1596</span><a name="line.1596"></a>
-<span class="sourceLineNo">1597</span> // allows to kill the executor before something is stored to the wal.<a name="line.1597"></a>
-<span class="sourceLineNo">1598</span> // useful to test the procedure recovery.<a name="line.1598"></a>
-<span class="sourceLineNo">1599</span> if (testing != null && testing.shouldKillBeforeStoreUpdate()) {<a name="line.1599"></a>
-<span class="sourceLineNo">1600</span> String msg = "TESTING: Kill before store update";<a name="line.1600"></a>
-<span class="sourceLineNo">1601</span> LOG.debug(msg);<a name="line.1601"></a>
-<span class="sourceLineNo">1602</span> stop();<a name="line.1602"></a>
-<span class="sourceLineNo">1603</span> throw new RuntimeException(msg);<a name="line.1603"></a>
-<span class="sourceLineNo">1604</span> }<a name="line.1604"></a>
-<span class="sourceLineNo">1605</span><a name="line.1605"></a>
-<span class="sourceLineNo">1606</span> cleanupAfterRollbackOneStep(proc);<a name="line.1606"></a>
-<span class="sourceLineNo">1607</span><a name="line.1607"></a>
-<span class="sourceLineNo">1608</span> return LockState.LOCK_ACQUIRED;<a name="line.1608"></a>
-<span class="sourceLineNo">1609</span> }<a name="line.1609"></a>
-<span class="sourceLineNo">1610</span><a name="line.1610"></a>
-<span class="sourceLineNo">1611</span> private void yieldProcedure(Procedure<TEnvironment> proc) {<a name="line.1611"></a>
-<span class="sourceLineNo">1612</span> releaseLock(proc, false);<a name="line.1612"></a>
-<span class="sourceLineNo">1613</span> scheduler.yield(proc);<a name="line.1613"></a>
-<span class="sourceLineNo">1614</span> }<a name="line.1614"></a>
-<span class="sourceLineNo">1615</span><a name="line.1615"></a>
-<span class="sourceLineNo">1616</span> /**<a name="line.1616"></a>
-<span class="sourceLineNo">1617</span> * Executes <code>procedure</code><a name="line.1617"></a>
-<span class="sourceLineNo">1618</span> * <ul><a name="line.1618"></a>
-<span class="sourceLineNo">1619</span> * <li>Calls the doExecute() of the procedure<a name="line.1619"></a>
-<span class="sourceLineNo">1620</span> * <li>If the procedure execution didn't fail (i.e. valid user input)<a name="line.1620"></a>
-<span class="sourceLineNo">1621</span> * <ul><a name="line.1621"></a>
-<span class="sourceLineNo">1622</span> * <li>...and returned subprocedures<a name="line.1622"></a>
-<span class="sourceLineNo">1623</span> * <ul><li>The subprocedures are initialized.<a name="line.1623"></a>
-<span class="sourceLineNo">1624</span> * <li>The subprocedures are added to the store<a name="line.1624"></a>
-<span class="sourceLineNo">1625</span> * <li>The subprocedures are added to the runnable queue<a name="line.1625"></a>
-<span class="sourceLineNo">1626</span> * <li>The procedure is now in a WAITING state, waiting for the subprocedures to complete<a name="line.1626"></a>
-<span class="sourceLineNo">1627</span> * </ul><a name="line.1627"></a>
-<span class="sourceLineNo">1628</span> * </li><a name="line.1628"></a>
-<span class="sourceLineNo">1629</span> * <li>...if there are no subprocedure<a name="line.1629"></a>
-<span class="sourceLineNo">1630</span> * <ul><li>the procedure completed successfully<a name="line.1630"></a>
-<span class="sourceLineNo">1631</span> * <li>if there is a parent (WAITING)<a name="line.1631"></a>
-<span class="sourceLineNo">1632</span> * <li>the parent state will be set to RUNNABLE<a name="line.1632"></a>
-<span class="sourceLineNo">1633</span> * </ul><a name="line.1633"></a>
-<span class="sourceLineNo">1634</span> * </li><a name="line.1634"></a>
-<span class="sourceLineNo">1635</span> * </ul><a name="line.1635"></a>
-<span class="sourceLineNo">1636</span> * </li><a name="line.1636"></a>
-<span class="sourceLineNo">1637</span> * <li>In case of failure<a name="line.1637"></a>
-<span class="sourceLineNo">1638</span> * <ul><a name="line.1638"></a>
-<span class="sourceLineNo">1639</span> * <li>The store is updated with the new state</li><a name="line.1639"></a>
-<span class="sourceLineNo">1640</span> * <li>The executor (caller of this method) will start the rollback of the procedure</li><a name="line.1640"></a>
-<span class="sourceLineNo">1641</span> * </ul><a name="line.1641"></a>
-<span class="sourceLineNo">1642</span> * </li><a name="line.1642"></a>
-<span class="sourceLineNo">1643</span> * </ul><a name="line.1643"></a>
-<span class="sourceLineNo">1644</span> */<a name="line.1644"></a>
-<span class="sourceLineNo">1645</span> private void execProcedure(RootProcedureState<TEnvironment> procStack,<a name="line.1645"></a>
-<span class="sourceLineNo">1646</span> Procedure<TEnvironment> procedure) {<a name="line.1646"></a>
-<span class="sourceLineNo">1647</span> Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.1647"></a>
-<span class="sourceLineNo">1648</span> "NOT RUNNABLE! " + procedure.toString());<a name="line.1648"></a>
-<span class="sourceLineNo">1649</span><a name="line.1649"></a>
-<span class="sourceLineNo">1650</span> // Procedures can suspend themselves. They skip out by throwing a ProcedureSuspendedException.<a name="line.1650"></a>
-<span class="sourceLineNo">1651</span> // The exception is caught below and then we hurry to the exit without disturbing state. The<a name="line.1651"></a>
-<span class="sourceLineNo">1652</span> // idea is that the processing of this procedure will be unsuspended later by an external event<a name="line.1652"></a>
-<span class="sourceLineNo">1653</span> // such the report of a region open.<a name="line.1653"></a>
-<span class="sourceLineNo">1654</span> boolean suspended = false;<a name="line.1654"></a>
-<span class="sourceLineNo">1655</span><a name="line.1655"></a>
-<span class="sourceLineNo">1656</span> // Whether to 're-' -execute; run through the loop again.<a name="line.1656"></a>
-<span class="sourceLineNo">1657</span> boolean reExecute = false;<a name="line.1657"></a>
-<span class="sourceLineNo">1658</span><a name="line.1658"></a>
-<span class="sourceLineNo">1659</span> Procedure<TEnvironment>[] subprocs = null;<a name="line.1659"></a>
-<span class="sourceLineNo">1660</span> do {<a name="line.1660"></a>
-<span class="sourceLineNo">1661</span> reExecute = false;<a name="line.1661"></a>
-<span class="sourceLineNo">1662</span> procedure.resetPersistence();<a name="line.1662"></a>
-<span class="sourceLineNo">1663</span> try {<a name="line.1663"></a>
-<span class="sourceLineNo">1664</span> subprocs = procedure.doExecute(getEnvironment());<a name="line.1664"></a>
-<span class="sourceLineNo">1665</span> if (subprocs != null && subprocs.length == 0) {<a name="line.1665"></a>
-<span class="sourceLineNo">1666</span> subprocs = null;<a name="line.1666"></a>
-<span class="sourceLineNo">1667</span> }<a name="line.1667"></a>
-<span class="sourceLineNo">1668</span> } catch (ProcedureSuspendedException e) {<a name="line.1668"></a>
-<span class="sourceLineNo">1669</span> LOG.trace("Suspend {}", procedure);<a name="line.1669"></a>
-<span class="sourceLineNo">1670</span> suspended = true;<a name="line.1670"></a>
-<span class="sourceLineNo">1671</span> } catch (ProcedureYieldException e) {<a name="line.1671"></a>
-<span class="sourceLineNo">1672</span> LOG.trace("Yield {}", procedure, e);<a name="line.1672"></a>
-<span class="sourceLineNo">1673</span> yieldProcedure(procedure);<a name="line.1673"></a>
-<span class="sourceLineNo">1674</span> return;<a name="line.1674"></a>
-<span class="sourceLineNo">1675</span> } catch (InterruptedException e) {<a name="line.1675"></a>
-<span class="sourceLineNo">1676</span> LOG.trace("Yield interrupt {}", procedure, e);<a name="line.1676"></a>
-<span class="sourceLineNo">1677</span> handleInterruptedException(procedure, e);<a name="line.1677"></a>
-<span class="sourceLineNo">1678</span> yieldProcedure(procedure);<a name="line.1678"></a>
-<span class="sourceLineNo">1679</span> return;<a name="line.1679"></a>
-<span class="sourceLineNo">1680</span> } catch (Throwable e) {<a name="line.1680"></a>
-<span class="sourceLineNo">1681</span> // Catch NullPointerExceptions or similar errors...<a name="line.1681"></a>
-<span class="sourceLineNo">1682</span> String msg = "CODE-BUG: Uncaught runtime exception: " + procedure;<a name="line.1682"></a>
-<span class="sourceLineNo">1683</span> LOG.error(msg, e);<a name="line.1683"></a>
-<span class="sourceLineNo">1684</span> procedure.setFailure(new RemoteProcedureException(msg, e));<a name="line.1684"></a>
-<span class="sourceLineNo">1685</span> }<a name="line.1685"></a>
-<span class="sourceLineNo">1686</span><a name="line.1686"></a>
-<span class="sourceLineNo">1687</span> if (!procedure.isFailed()) {<a name="line.1687"></a>
-<span class="sourceLineNo">1688</span> if (subprocs != null) {<a name="line.1688"></a>
-<span class="sourceLineNo">1689</span> if (subprocs.length == 1 && subprocs[0] == procedure) {<a name="line.1689"></a>
-<span class="sourceLineNo">1690</span> // Procedure returned itself. Quick-shortcut for a state machine-like procedure;<a name="line.1690"></a>
-<span class="sourceLineNo">1691</span> // i.e. we go around this loop again rather than go back out on the scheduler queue.<a name="line.1691"></a>
-<span class="sourceLineNo">1692</span> subprocs = null;<a name="line.1692"></a>
-<span class="sourceLineNo">1693</span> reExecute = true;<a name="line.1693"></a>
-<span class="sourceLineNo">1694</span> LOG.trace("Short-circuit to next step on pid={}", procedure.getProcId());<a name="line.1694"></a>
-<span class="sourceLineNo">1695</span> } else {<a name="line.1695"></a>
-<span class="sourceLineNo">1696</span> // Yield the current procedure, and make the subprocedure runnable<a name="line.1696"></a>
-<span class="sourceLineNo">1697</span> // subprocs may come back 'null'.<a name="line.1697"></a>
-<span class="sourceLineNo">1698</span> subprocs = initializeChildren(procStack, procedure, subprocs);<a name="line.1698"></a>
-<span class="sourceLineNo">1699</span> LOG.info("Initialized subprocedures=" +<a name="line.1699"></a>
-<span class="sourceLineNo">1700</span> (subprocs == null? null:<a name="line.1700"></a>
-<span class="sourceLineNo">1701</span> Stream.of(subprocs).map(e -> "{" + e.toString() + "}").<a name="line.1701"></a>
-<span class="sourceLineNo">1702</span> collect(Collectors.toList()).toString()));<a name="line.1702"></a>
-<span class="sourceLineNo">1703</span> }<a name="line.1703"></a>
-<span class="sourceLineNo">1704</span> } else if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.1704"></a>
-<span class="sourceLineNo">1705</span> LOG.trace("Added to timeoutExecutor {}", procedure);<a name="line.1705"></a>
-<span class="sourceLineNo">1706</span> timeoutExecutor.add(procedure);<a name="line.1706"></a>
-<span class="sourceLineNo">1707</span> } else if (!suspended) {<a name="line.1707"></a>
-<span class="sourceLineNo">1708</span> // No subtask, so we are done<a name="line.1708"></a>
-<span class="sourceLineNo">1709</span> procedure.setState(ProcedureState.SUCCESS);<a name="line.1709"></a>
-<span class="sourceLineNo">1710</span> }<a name="line.1710"></a>
-<span class="sourceLineNo">1711</span> }<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span><a name="line.1712"></a>
-<span class="sourceLineNo">1713</span> // Add the procedure to the stack<a name="line.1713"></a>
-<span class="sourceLineNo">1714</span> procStack.addRollbackStep(procedure);<a name="line.1714"></a>
-<span class="sourceLineNo">1715</span><a name="line.1715"></a>
-<span class="sourceLineNo">1716</span> // allows to kill the executor before something is stored to the wal.<a name="line.1716"></a>
-<span class="sourceLineNo">1717</span> // useful to test the procedure recovery.<a name="line.1717"></a>
-<span class="sourceLineNo">1718</span> if (testing != null &&<a name="line.1718"></a>
-<span class="sourceLineNo">1719</span> testing.shouldKillBeforeStoreUpdate(suspended, procedure.hasParent())) {<a name="line.1719"></a>
-<span class="sourceLineNo">1720</span> kill("TESTING: Kill BEFORE store update: " + procedure);<a name="line.1720"></a>
-<span class="sourceLineNo">1721</span> }<a name="line.1721"></a>
+<span class="sourceLineNo">1553</span> // Finalize the procedure state<a name="line.1553"></a>
+<span class="sourceLineNo">1554</span> LOG.info("Rolled back {} exec-time={}", rootProc,<a name="line.1554"></a>
+<span class="sourceLineNo">1555</span> StringUtils.humanTimeDiff(rootProc.elapsedTime()));<a name="line.1555"></a>
+<span class="sourceLineNo">1556</span> procedureFinished(rootProc);<a name="line.1556"></a>
+<span class="sourceLineNo">1557</span> return LockState.LOCK_ACQUIRED;<a name="line.1557"></a>
+<span class="sourceLineNo">1558</span> }<a name="line.1558"></a>
+<span class="sourceLineNo">1559</span><a name="line.1559"></a>
+<span class="sourceLineNo">1560</span> private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) {<a name="line.1560"></a>
+<span class="sourceLineNo">1561</span> if (proc.removeStackIndex()) {<a name="line.1561"></a>
+<span class="sourceLineNo">1562</span> if (!proc.isSuccess()) {<a name="line.1562"></a>
+<span class="sourceLineNo">1563</span> proc.setState(ProcedureState.ROLLEDBACK);<a name="line.1563"></a>
+<span class="sourceLineNo">1564</span> }<a name="line.1564"></a>
+<span class="sourceLineNo">1565</span><a name="line.1565"></a>
+<span class="sourceLineNo">1566</span> // update metrics on finishing the procedure (fail)<a name="line.1566"></a>
+<span class="sourceLineNo">1567</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false);<a name="line.1567"></a>
+<span class="sourceLineNo">1568</span><a name="line.1568"></a>
+<span class="sourceLineNo">1569</span> if (proc.hasParent()) {<a name="line.1569"></a>
+<span class="sourceLineNo">1570</span> store.delete(proc.getProcId());<a name="line.1570"></a>
+<span class="sourceLineNo">1571</span> procedures.remove(proc.getProcId());<a name="line.1571"></a>
+<span class="sourceLineNo">1572</span> } else {<a name="line.1572"></a>
+<span class="sourceLineNo">1573</span> final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds();<a name="line.1573"></a>
+<span class="sourceLineNo">1574</span> if (childProcIds != null) {<a name="line.1574"></a>
+<span class="sourceLineNo">1575</span> store.delete(proc, childProcIds);<a name="line.1575"></a>
+<span class="sourceLineNo">1576</span> } else {<a name="line.1576"></a>
+<span class="sourceLineNo">1577</span> store.update(proc);<a name="line.1577"></a>
+<span class="sourceLineNo">1578</span> }<a name="line.1578"></a>
+<span class="sourceLineNo">1579</span> }<a name="line.1579"></a>
+<span class="sourceLineNo">1580</span> } else {<a name="line.1580"></a>
+<span class="sourceLineNo">1581</span> store.update(proc);<a name="line.1581"></a>
+<span class="sourceLineNo">1582</span> }<a name="line.1582"></a>
+<span class="sourceLineNo">1583</span> }<a name="line.1583"></a>
+<span class="sourceLineNo">1584</span><a name="line.1584"></a>
+<span class="sourceLineNo">1585</span> /**<a name="line.1585"></a>
+<span class="sourceLineNo">1586</span> * Execute the rollback of the procedure step.<a name="line.1586"></a>
+<span class="sourceLineNo">1587</span> * It updates the store with the new state (stack index)<a name="line.1587"></a>
+<span class="sourceLineNo">1588</span> * or will remove completly the procedure in case it is a child.<a name="line.1588"></a>
+<span class="sourceLineNo">1589</span> */<a name="line.1589"></a>
+<span class="sourceLineNo">1590</span> private LockState executeRollback(Procedure<TEnvironment> proc) {<a name="line.1590"></a>
+<span class="sourceLineNo">1591</span> try {<a name="line.1591"></a>
+<span class="sourceLineNo">1592</span> proc.doRollback(getEnvironment());<a name="line.1592"></a>
+<span class="sourceLineNo">1593</span> } catch (IOException e) {<a name="line.1593"></a>
+<span class="sourceLineNo">1594</span> LOG.debug("Roll back attempt failed for {}", proc, e);<a name="line.1594"></a>
+<span class="sourceLineNo">1595</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1595"></a>
+<span class="sourceLineNo">1596</span> } catch (InterruptedException e) {<a name="line.1596"></a>
+<span class="sourceLineNo">1597</span> handleInterruptedException(proc, e);<a name="line.1597"></a>
+<span class="sourceLineNo">1598</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1598"></a>
+<span class="sourceLineNo">1599</span> } catch (Throwable e) {<a name="line.1599"></a>
+<span class="sourceLineNo">1600</span> // Catch NullPointerExceptions or similar errors...<a name="line.1600"></a>
+<span class="sourceLineNo">1601</span> LOG.error(HBaseMarkers.FATAL, "CODE-BUG: Uncaught runtime exception for " + proc, e);<a name="line.1601"></a>
+<span class="sourceLineNo">1602</span> }<a name="line.1602"></a>
+<span class="sourceLineNo">1603</span><a name="line.1603"></a>
+<span class="sourceLineNo">1604</span> // allows to kill the executor before something is stored to the wal.<a name="line.1604"></a>
+<span class="sourceLineNo">1605</span> // useful to test the procedure recovery.<a name="line.1605"></a>
+<span class="sourceLineNo">1606</span> if (testing != null && testing.shouldKillBeforeStoreUpdate()) {<a name="line.1606"></a>
+<span class="sourceLineNo">1607</span> String msg = "TESTING: Kill before store update";<a name="line.1607"></a>
+<span class="sourceLineNo">1608</span> LOG.debug(msg);<a name="line.1608"></a>
+<span class="sourceLineNo">1609</span> stop();<a name="line.1609"></a>
+<span class="sourceLineNo">1610</span> throw new RuntimeException(msg);<a name="line.1610"></a>
+<span class="sourceLineNo">1611</span> }<a name="line.1611"></a>
+<span class="sourceLineNo">1612</span><a name="line.1612"></a>
+<span class="sourceLineNo">1613</span> cleanupAfterRollbackOneStep(proc);<a name="line.1613"></a>
+<span class="sourceLineNo">1614</span><a name="line.1614"></a>
+<span class="sourceLineNo">1615</span> return LockState.LOCK_ACQUIRED;<a name="line.1615"></a>
+<span class="sourceLineNo">1616</span> }<a name="line.1616"></a>
+<span class="sourceLineNo">1617</span><a name="line.1617"></a>
+<span class="sourceLineNo">1618</span> private void yieldProcedure(Procedure<TEnvironment> proc) {<a name="line.1618"></a>
+<span class="sourceLineNo">1619</span> releaseLock(proc, false);<a name="line.1619"></a>
+<span class="sourceLineNo">1620</span> scheduler.yield(proc);<a name="line.1620"></a>
+<span class="sourceLineNo">1621</span> }<a name="line.1621"></a>
+<span class="sourceLineNo">1622</span><a name="line.1622"></a>
+<span class="sourceLineNo">1623</span> /**<a name="line.1623"></a>
+<span class="sourceLineNo">1624</span> * Executes <code>procedure</code><a name="line.1624"></a>
+<span class="sourceLineNo">1625</span> * <ul><a name="line.1625"></a>
+<span class="sourceLineNo">1626</span> * <li>Calls the doExecute() of the procedure<a name="line.1626"></a>
+<span class="sourceLineNo">1627</span> * <li>If the procedure execution didn't fail (i.e. valid user input)<a name="line.1627"></a>
+<span class="sourceLineNo">1628</span> * <ul><a name="line.1628"></a>
+<span class="sourceLineNo">1629</span> * <li>...and returned subprocedures<a name="line.1629"></a>
+<span class="sourceLineNo">1630</span> * <ul><li>The subprocedures are initialized.<a name="line.1630"></a>
+<span class="sourceLineNo">1631</span> * <li>The subprocedures are added to the store<a name="line.1631"></a>
+<span class="sourceLineNo">1632</span> * <li>The subprocedures are added to the runnable queue<a name="line.1632"></a>
+<span class="sourceLineNo">1633</span> * <li>The procedure is now in a WAITING state, waiting for the subprocedures to complete<a name="line.1633"></a>
+<span class="sourceLineNo">1634</span> * </ul><a name="line.1634"></a>
+<span class="sourceLineNo">1635</span> * </li><a name="line.1635"></a>
+<span class="sourceLineNo">1636</span> * <li>...if there are no subprocedure<a name="line.1636"></a>
+<span class="sourceLineNo">1637</span> * <ul><li>the procedure completed successfully<a name="line.1637"></a>
+<span class="sourceLineNo">1638</span> * <li>if there is a parent (WAITING)<a name="line.1638"></a>
+<span class="sourceLineNo">1639</span> * <li>the parent state will be set to RUNNABLE<a name="line.1639"></a>
+<span class="sourceLineNo">1640</span> * </ul><a name="line.1640"></a>
+<span class="sourceLineNo">1641</span> * </li><a name="line.1641"></a>
+<span class="sourceLineNo">1642</span> * </ul><a name="line.1642"></a>
+<span class="sourceLineNo">1643</span> * </li><a name="line.1643"></a>
+<span class="sourceLineNo">1644</span> * <li>In case of failure<a name="line.1644"></a>
+<span class="sourceLineNo">1645</span> * <ul><a name="line.1645"></a>
+<span class="sourceLineNo">1646</span> * <li>The store is updated with the new state</li><a name="line.1646"></a>
+<span class="sourceLineNo">1647</span> * <li>The executor (caller of this method) will start the rollback of the procedure</li><a name="line.1647"></a>
+<span class="sourceLineNo">1648</span> * </ul><a name="line.1648"></a>
+<span class="sourceLineNo">1649</span> * </li><a name="line.1649"></a>
+<span class="sourceLineNo">1650</span> * </ul><a name="line.1650"></a>
+<span class="sourceLineNo">1651</span> */<a name="line.1651"></a>
+<span class="sourceLineNo">1652</span> private void execProcedure(RootProcedureState<TEnvironment> procStack,<a name="line.1652"></a>
+<span class="sourceLineNo">1653</span> Procedure<TEnvironment> procedure) {<a name="line.1653"></a>
+<span class="sourceLineNo">1654</span> Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.1654"></a>
+<span class="sourceLineNo">1655</span> "NOT RUNNABLE! " + procedure.toString());<a name="line.1655"></a>
+<span class="sourceLineNo">1656</span><a name="line.1656"></a>
+<span class="sourceLineNo">1657</span> // Procedures can suspend themselves. They skip out by throwing a ProcedureSuspendedException.<a name="line.1657"></a>
+<span class="sourceLineNo">1658</span> // The exception is caught below and then we hurry to the exit without disturbing state. The<a name="line.1658"></a>
+<span class="sourceLineNo">1659</span> // idea is that the processing of this procedure will be unsuspended later by an external event<a name="line.1659"></a>
+<span class="sourceLineNo">1660</span> // such the report of a region open.<a name="line.1660"></a>
+<span class="sourceLineNo">1661</span> boolean suspended = false;<a name="line.1661"></a>
+<span class="sourceLineNo">1662</span><a name="line.1662"></a>
+<span class="sourceLineNo">1663</span> // Whether to 're-' -execute; run through the loop again.<a name="line.1663"></a>
+<span class="sourceLineNo">1664</span> boolean reExecute = false;<a name="line.1664"></a>
+<span class="sourceLineNo">1665</span><a name="line.1665"></a>
+<span class="sourceLineNo">1666</span> Procedure<TEnvironment>[] subprocs = null;<a name="line.1666"></a>
+<span class="sourceLineNo">1667</span> do {<a name="line.1667"></a>
+<span class="sourceLineNo">1668</span> reExecute = false;<a name="line.1668"></a>
+<span class="sourceLineNo">1669</span> procedure.resetPersistence();<a name="line.1669"></a>
+<span class="sourceLineNo">1670</span> try {<a name="line.1670"></a>
+<span class="sourceLineNo">1671</span> subprocs = procedure.doExecute(getEnvironment());<a name="line.1671"></a>
+<span class="sourceLineNo">1672</span> if (subprocs != null && subprocs.length == 0) {<a name="line.1672"></a>
+<span class="sourceLineNo">1673</span> subprocs = null;<a name="line.1673"></a>
+<span class="sourceLineNo">1674</span> }<a name="line.1674"></a>
+<span class="sourceLineNo">1675</span> } catch (ProcedureSuspendedException e) {<a name="line.1675"></a>
+<span class="sourceLineNo">1676</span> LOG.trace("Suspend {}", procedure);<a name="line.1676"></a>
+<span class="sourceLineNo">1677</span> suspended = true;<a name="line.1677"></a>
+<span class="sourceLineNo">1678</span> } catch (ProcedureYieldException e) {<a name="line.1678"></a>
+<span class="sourceLineNo">1679</span> LOG.trace("Yield {}", procedure, e);<a name="line.1679"></a>
+<span class="sourceLineNo">1680</span> yieldProcedure(procedure);<a name="line.1680"></a>
+<span class="sourceLineNo">1681</span> return;<a name="line.1681"></a>
+<span class="sourceLineNo">1682</span> } catch (InterruptedException e) {<a name="line.1682"></a>
+<span class="sourceLineNo">1683</span> LOG.trace("Yield interrupt {}", procedure, e);<a name="line.1683"></a>
+<span class="sourceLineNo">1684</span> handleInterruptedException(procedure, e);<a name="line.1684"></a>
+<span class="sourceLineNo">1685</span> yieldProcedure(procedure);<a name="line.1685"></a>
+<span class="sourceLineNo">1686</span> return;<a name="line.1686"></a>
+<span class="sourceLineNo">1687</span> } catch (Throwable e) {<a name="line.1687"></a>
+<span class="sourceLineNo">1688</span> // Catch NullPointerExceptions or similar errors...<a name="line.1688"></a>
+<span class="sourceLineNo">1689</span> String msg = "CODE-BUG: Uncaught runtime exception: " + procedure;<a name="line.1689"></a>
+<span class="sourceLineNo">1690</span> LOG.error(msg, e);<a name="line.1690"></a>
+<span class="sourceLineNo">1691</span> procedure.setFailure(new RemoteProcedureException(msg, e));<a name="line.1691"></a>
+<span class="sourceLineNo">1692</span> }<a name="line.1692"></a>
+<span class="sourceLineNo">1693</span><a name="line.1693"></a>
+<span class="sourceLineNo">1694</span> if (!procedure.isFailed()) {<a name="line.1694"></a>
+<span class="sourceLineNo">1695</span> if (subprocs != null) {<a name="line.1695"></a>
+<span class="sourceLineNo">1696</span> if (subprocs.length == 1 && subprocs[0] == procedure) {<a name="line.1696"></a>
+<span class="sourceLineNo">1697</span> // Procedure returned itself. Quick-shortcut for a state machine-like procedure;<a name="line.1697"></a>
+<span class="sourceLineNo">1698</span> // i.e. we go around this loop again rather than go back out on the scheduler queue.<a name="line.1698"></a>
+<span class="sourceLineNo">1699</span> subprocs = null;<a name="line.1699"></a>
+<span class="sourceLineNo">1700</span> reExecute = true;<a name="line.1700"></a>
+<span class="sourceLineNo">1701</span> LOG.trace("Short-circuit to next step on pid={}", procedure.getProcId());<a name="line.1701"></a>
+<span class="sourceLineNo">1702</span> } else {<a name="line.1702"></a>
+<span class="sourceLineNo">1703</span> // Yield the current procedure, and make the subprocedure runnable<a name="line.1703"></a>
+<span class="sourceLineNo">1704</span> // subprocs may come back 'null'.<a name="line.1704"></a>
+<span class="sourceLineNo">1705</span> subprocs = initializeChildren(procStack, procedure, subprocs);<a name="line.1705"></a>
+<span class="sourceLineNo">1706</span> LOG.info("Initialized subprocedures=" +<a name="line.1706"></a>
+<span class="sourceLineNo">1707</span> (subprocs == null? null:<a name="line.1707"></a>
+<span class="sourceLineNo">1708</span> Stream.of(subprocs).map(e -> "{" + e.toString() + "}").<a name="line.1708"></a>
+<span class="sourceLineNo">1709</span> collect(Collectors.toList()).toString()));<a name="line.1709"></a>
+<span class="sourceLineNo">1710</span> }<a name="line.1710"></a>
+<span class="sourceLineNo">1711</span> } else if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.1711"></a>
+<span class="sourceLineNo">1712</span> LOG.trace("Added to timeoutExecutor {}", procedure);<a name="line.1712"></a>
+<span class="sourceLineNo">1713</span> timeoutExecutor.add(procedure);<a name="line.1713"></a>
+<span class="sourceLineNo">1714</span> } else if (!suspended) {<a name="line.1714"></a>
+<span class="sourceLineNo">1715</span> // No subtask, so we are done<a name="line.1715"></a>
+<span class="sourceLineNo">1716</span> procedure.setState(ProcedureState.SUCCESS);<a name="line.1716"></a>
+<span class="sourceLineNo">1717</span> }<a name="line.1717"></a>
+<span class="sourceLineNo">1718</span> }<a name="line.1718"></a>
+<span class="sourceLineNo">1719</span><a name="line.1719"></a>
+<span class="sourceLineNo">1720</span> // Add the procedure to the stack<a name="line.1720"></a>
+<span class="sourceLineNo">1721</span> procStack.addRollbackStep(procedure);<a name="line.1721"></a>
<span class="sourceLineNo">1722</span><a name="line.1722"></a>
-<span class="sourceLineNo">1723</span> // TODO: The code here doesn't check if store is running before persisting to the store as<a name="line.1723"></a>
-<span class="sourceLineNo">1724</span> // it relies on the method call below to throw RuntimeException to wind up the stack and<a name="line.1724"></a>
-<span class="sourceLineNo">1725</span> // executor thread to stop. The statement following the method call below seems to check if<a name="line.1725"></a>
-<span class="sourceLineNo">1726</span> // store is not running, to prevent scheduling children procedures, re-execution or yield<a name="line.1726"></a>
-<span class="sourceLineNo">1727</span> // of this procedure. This may need more scrutiny and subsequent cleanup in future<a name="line.1727"></a>
-<span class="sourceLineNo">1728</span> //<a name="line.1728"></a>
-<span class="sourceLineNo">1729</span> // Commit the transaction even if a suspend (state may have changed). Note this append<a name="line.1729"></a>
-<span class="sourceLineNo">1730</span> // can take a bunch of time to complete.<a name="line.1730"></a>
-<span class="sourceLineNo">1731</span> if (procedure.needPersistence()) {<a name="line.1731"></a>
-<span class="sourceLineNo">1732</span> updateStoreOnExec(procStack, procedure, subprocs);<a name="line.1732"></a>
-<span class="sourceLineNo">1733</span> }<a name="line.1733"></a>
-<span class="sourceLineNo">1734</span><a name="line.1734"></a>
-<span class="sourceLineNo">1735</span> // if the store is not running we are aborting<a name="line.1735"></a>
-<span class="sourceLineNo">1736</span> if (!store.isRunning()) {<a name="line.1736"></a>
-<span class="sourceLineNo">1737</span> return;<a name="line.1737"></a>
-<span class="sourceLineNo">1738</span> }<a name="line.1738"></a>
-<span class="sourceLineNo">1739</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1739"></a>
-<span class="sourceLineNo">1740</span> if (procedure.isRunnable() && !suspended &&<a name="line.1740"></a>
-<span class="sourceLineNo">1741</span> procedure.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1741"></a>
-<span class="sourceLineNo">1742</span> yieldProcedure(procedure);<a name="line.1742"></a>
-<span class="sourceLineNo">1743</span> return;<a name="line.1743"></a>
-<span class="sourceLineNo">1744</span> }<a name="line.1744"></a>
-<span class="sourceLineNo">1745</span><a name="line.1745"></a>
-<span class="sourceLineNo">1746</span> assert (reExecute && subprocs == null) || !reExecute;<a name="line.1746"></a>
-<span class="sourceLineNo">1747</span> } while (reExecute);<a name="line.1747"></a>
-<span class="sourceLineNo">1748</span><a name="line.1748"></a>
-<span class="sourceLineNo">1749</span> // Allows to kill the executor after something is stored to the WAL but before the below<a name="line.1749"></a>
-<span class="sourceLineNo">1750</span> // state settings are done -- in particular the one on the end where we make parent<a name="line.1750"></a>
-<span class="sourceLineNo">1751</span> // RUNNABLE again when its children are done; see countDownChildren.<a name="line.1751"></a>
-<span class="sourceLineNo">1752</span> if (testing != null && testing.shouldKillAfterStoreUpdate(suspended)) {<a name="line.1752"></a>
-<span class="sourceLineNo">1753</span> kill("TESTING: Kill AFTER store update: " + procedure);<a name="line.1753"></a>
-<span class="sourceLineNo">1754</span> }<a name="line.1754"></a>
+<span class="sourceLineNo">1723</span> // allows to kill the executor before something is stored to the wal.<a name="line.1723"></a>
+<span class="sourceLineNo">1724</span> // useful to test the procedure recovery.<a name="line.1724"></a>
+<span class="sourceLineNo">1725</span> if (testing != null &&<a name="line.1725"></a>
+<span class="sourceLineNo">1726</span> testing.shouldKillBeforeStoreUpdate(suspended, procedure.hasParent())) {<a name="line.1726"></a>
+<span class="sourceLineNo">1727</span> kill("TESTING: Kill BEFORE store update: " + procedure);<a name="line.1727"></a>
+<span class="sourceLineNo">1728</span> }<a name="line.1728"></a>
+<span class="sourceLineNo">1729</span><a name="line.1729"></a>
+<span class="sourceLineNo">1730</span> // TODO: The code here doesn't check if store is running before persisting to the store as<a name="line.1730"></a>
+<span class="sourceLineNo">1731</span> // it relies on the method call below to throw RuntimeException to wind up the stack and<a name="line.1731"></a>
+<span class="sourceLineNo">1732</span> // executor thread to stop. The statement following the method call below seems to check if<a name="line.1732"></a>
+<span class="sourceLineNo">1733</span> // store is not running, to prevent scheduling children procedures, re-execution or yield<a name="line.1733"></a>
+<span class="sourceLineNo">1734</span> // of this procedure. This may need more scrutiny and subsequent cleanup in future<a name="line.1734"></a>
+<span class="sourceLineNo">1735</span> //<a name="line.1735"></a>
+<span class="sourceLineNo">1736</span> // Commit the transaction even if a suspend (state may have changed). Note this append<a name="line.1736"></a>
+<span class="sourceLineNo">1737</span> // can take a bunch of time to complete.<a name="line.1737"></a>
+<span class="sourceLineNo">1738</span> if (procedure.needPersistence()) {<a name="line.1738"></a>
+<span class="sourceLineNo">1739</span> updateStoreOnExec(procStack, procedure, subprocs);<a name="line.1739"></a>
+<span class="sourceLineNo">1740</span> }<a name="line.1740"></a>
+<span class="sourceLineNo">1741</span><a name="line.1741"></a>
+<span class="sourceLineNo">1742</span> // if the store is not running we are aborting<a name="line.1742"></a>
+<span class="sourceLineNo">1743</span> if (!store.isRunning()) {<a name="line.1743"></a>
+<span class="sourceLineNo">1744</span> return;<a name="line.1744"></a>
+<span class="sourceLineNo">1745</span> }<a name="line.1745"></a>
+<span class="sourceLineNo">1746</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1746"></a>
+<span class="sourceLineNo">1747</span> if (procedure.isRunnable() && !suspended &&<a name="line.1747"></a>
+<span class="sourceLineNo">1748</span> procedure.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1748"></a>
+<span class="sourceLineNo">1749</span> yieldProcedure(procedure);<a name="line.1749"></a>
+<span class="sourceLineNo">1750</span> return;<a name="line.1750"></a>
+<span class="sourceLineNo">1751</span> }<a name="line.1751"></a>
+<span class="sourceLineNo">1752</span><a name="line.1752"></a>
+<span class="sourceLineNo">1753</span> assert (reExecute && subprocs == null) || !reExecute;<a name="line.1753"></a>
+<span class="sourceLineNo">1754</span> } while (reExecute);<a name="line.1754"></a>
<span class="sourceLineNo">1755</span><a name="line.1755"></a>
-<span class="sourceLineNo">1756</span> // Submit the new subprocedures<a name="line.1756"></a>
-<span class="sourceLineNo">1757</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1757"></a>
-<span class="sourceLineNo">1758</span> submitChildrenProcedures(subprocs);<a name="line.1758"></a>
-<span class="sourceLineNo">1759</span> }<a name="line.1759"></a>
-<span class="sourceLineNo">1760</span><a name="line.1760"></a>
-<span class="sourceLineNo">1761</span> // we need to log the release lock operation before waking up the parent procedure, as there<a name="line.1761"></a>
-<span class="sourceLineNo">1762</span> // could be race that the parent procedure may call updateStoreOnExec ahead of us and remove all<a name="line.1762"></a>
-<span class="sourceLineNo">1763</span> // the sub procedures from store and cause problems...<a name="line.1763"></a>
-<span class="sourceLineNo">1764</span> releaseLock(procedure, false);<a name="line.1764"></a>
-<span class="sourceLineNo">1765</span><a name="line.1765"></a>
-<span class="sourceLineNo">1766</span> // if the procedure is complete and has a parent, count down the children latch.<a name="line.1766"></a>
-<span class="sourceLineNo">1767</span> // If 'suspended', do nothing to change state -- let other threads handle unsuspend event.<a name="line.1767"></a>
-<span class="sourceLineNo">1768</span> if (!suspended && procedure.isFinished() && procedure.hasParent()) {<a name="line.1768"></a>
-<span class="sourceLineNo">1769</span> countDownChildren(procStack, procedure);<a name="line.1769"></a>
-<span class="sourceLineNo">1770</span> }<a name="line.1770"></a>
-<span class="sourceLineNo">1771</span> }<a name="line.1771"></a>
+<span class="sourceLineNo">1756</span> // Allows to kill the executor after something is stored to the WAL but before the below<a name="line.1756"></a>
+<span class="sourceLineNo">1757</span> // state settings are done -- in particular the one on the end where we make parent<a name="line.1757"></a>
+<span class="sourceLineNo">1758</span> // RUNNABLE again when its children are done; see countDownChildren.<a name="line.1758"></a>
+<span class="sourceLineNo">1759</span> if (testing != null && testing.shouldKillAfterStoreUpdate(suspended)) {<a name="line.1759"></a>
+<span class="sourceLineNo">1760</span> kill("TESTING: Kill AFTER store update: " + procedure);<a name="line.1760"></a>
+<span class="sourceLineNo">1761</span> }<a name="line.1761"></a>
+<span class="sourceLineNo">1762</span><a name="line.1762"></a>
+<span class="sourceLineNo">1763</span> // Submit the new subprocedures<a name="line.1763"></a>
+<span class="sourceLineNo">1764</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1764"></a>
+<span class="sourceLineNo">1765</span> submitChildrenProcedures(subprocs);<a name="line.1765"></a>
+<span class="sourceLineNo">1766</span> }<a name="line.1766"></a>
+<span class="sourceLineNo">1767</span><a name="line.1767"></a>
+<span class="sourceLineNo">1768</span> // we need to log the release lock operation before waking up the parent procedure, as there<a name="line.1768"></a>
+<span class="sourceLineNo">1769</span> // could be race that the parent procedure may call updateStoreOnExec ahead of us and remove all<a name="line.1769"></a>
+<span class="sourceLineNo">1770</span> // the sub procedures from store and cause problems...<a name="line.1770"></a>
+<span class="sourceLineNo">1771</span> releaseLock(procedure, false);<a name="line.1771"></a>
<span class="sourceLineNo">1772</span><a name="line.1772"></a>
-<span class="sourceLineNo">1773</span> private void kill(String msg) {<a name="line.1773"></a>
-<span class="sourceLineNo">1774</span> LOG.debug(msg);<a name="line.1774"></a>
-<span class="sourceLineNo">1775</span> stop();<a name="line.1775"></a>
-<span class="sourceLineNo">1776</span> throw new RuntimeException(msg);<a name="line.1776"></a>
-<span class="sourceLineNo">1777</span> }<a name="line.1777"></a>
-<span class="sourceLineNo">1778</span><a name="line.1778"></a>
-<span class="sourceLineNo">1779</span> private Procedure<TEnvironment>[] initializeChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1779"></a>
-<span class="sourceLineNo">1780</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1780"></a>
-<span class="sourceLineNo">1781</span> assert subprocs != null : "expected subprocedures";<a name="line.1781"></a>
-<span class="sourceLineNo">1782</span> final long rootProcId = getRootProcedureId(procedure);<a name="line.1782"></a>
-<span class="sourceLineNo">1783</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1784"></a>
-<span class="sourceLineNo">1785</span> if (subproc == null) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span> String msg = "subproc[" + i + "] is null, aborting the procedure";<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span> procedure.setFailure(new RemoteProcedureException(msg,<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span> new IllegalArgumentIOException(msg)));<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span> return null;<a name="line.1789"></a>
-<span class="sourceLineNo">1790</span> }<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span><a name="line.1791"></a>
-<span class="sourceLineNo">1792</span> assert subproc.getState() == ProcedureState.INITIALIZING : subproc;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span> subproc.setParentProcId(procedure.getProcId());<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span> subproc.setRootProcId(rootProcId);<a name="line.1794"></a>
-<span class="sourceLineNo">1795</span> subproc.setProcId(nextProcId());<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span> procStack.addSubProcedure(subproc);<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span> }<a name="line.1797"></a>
+<span class="sourceLineNo">1773</span> // if the procedure is complete and has a parent, count down the children latch.<a name="line.1773"></a>
+<span class="sourceLineNo">1774</span> // If 'suspended', do nothing to change state -- let other threads handle unsuspend event.<a name="line.1774"></a>
+<span class="sourceLineNo">1775</span> if (!suspended && procedure.isFinished() && procedure.hasParent()) {<a name="line.1775"></a>
+<span class="sourceLineNo">1776</span> countDownChildren(procStack, procedure);<a name="line.1776"></a>
+<span class="sourceLineNo">1777</span> }<a name="line.1777"></a>
+<span class="sourceLineNo">1778</span> }<a name="line.1778"></a>
+<span class="sourceLineNo">1779</span><a name="line.1779"></a>
+<span class="sourceLineNo">1780</span> private void kill(String msg) {<a name="line.1780"></a>
+<span class="sourceLineNo">1781</span> LOG.debug(msg);<a name="line.1781"></a>
+<span class="sourceLineNo">1782</span> stop();<a name="line.1782"></a>
+<span class="sourceLineNo">1783</span> throw new RuntimeException(msg);<a name="line.1783"></a>
+<span class="sourceLineNo">1784</span> }<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span><a name="line.1785"></a>
+<span class="sourceLineNo">1786</span> private Procedure<TEnvironment>[] initializeChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span> assert subprocs != null : "expected subprocedures";<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span> final long rootProcId = getRootProcedureId(procedure);<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span> if (subproc == null) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span> String msg = "subproc[" + i + "] is null, aborting the procedure";<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span> procedure.setFailure(new RemoteProcedureException(msg,<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span> new IllegalArgumentIOException(msg)));<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span> return null;<a name="line.1796"></a>
+<span class="sourceLineNo">1797</span> }<a name="line.1797"></a>
<span class="sourceLineNo">1798</span><a name="line.1798"></a>
-<span class="sourceLineNo">1799</span> if (!procedure.isFailed()) {<a name="line.1799"></a>
-<span class="sourceLineNo">1800</span> procedure.setChildrenLatch(subprocs.length);<a name="line.1800"></a>
-<span class="sourceLineNo">1801</span> switch (procedure.getState()) {<a name="line.1801"></a>
-<span class="sourceLineNo">1802</span> case RUNNABLE:<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span> procedure.setState(ProcedureState.WAITING);<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span> break;<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span> case WAITING_TIMEOUT:<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span> timeoutExecutor.add(procedure);<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span> break;<a name="line.1807"></a>
-<span class="sourceLineNo">1808</span> default:<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span> break;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span> }<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span> }<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span> return subprocs;<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span> }<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span><a name="line.1814"></a>
-<span class="sourceLineNo">1815</span> private void submitChildrenProcedures(Procedure<TEnvironment>[] subprocs) {<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span> subproc.updateMetricsOnSubmit(getEnvironment());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span> assert !procedures.containsKey(subproc.getProcId());<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span> procedures.put(subproc.getProcId(), subproc);<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span> scheduler.addFront(subproc);<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span> }<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span> }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span><a name="line.1824"></a>
-<span class="sourceLineNo">1825</span> private void countDownChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span> Procedure<TEnvironment> procedure) {<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span> Procedure<TEnvironment> parent = procedures.get(procedure.getParentProcId());<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span> if (parent == null) {<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span> assert procStack.isRollingback();<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span> return;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span> }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span> // If this procedure is the last child awake the parent procedure<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span> if (parent.tryRunnable()) {<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span> // If we succeeded in making the parent runnable -- i.e. all of its<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span> // children have completed, move parent to front of the queue.<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span> store.update(parent);<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span> scheduler.addFront(parent);<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span> LOG.info("Finished subprocedure(s) of " + parent + "; resume parent processing.");<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span> return;<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span> }<a name="line.1841"></a>
-<span class="sourceLineNo">1842</span> }<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span><a name="line.1843"></a>
-<span class="sourceLineNo">1844</span> private void updateStoreOnExec(RootProcedureState<TEnvironment> procStack,<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span> if (LOG.isTraceEnabled()) {<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span> LOG.trace("Stored " + procedure + ", children " + Arrays.toString(subprocs));<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span> }<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span> store.insert(procedure, subprocs);<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span> } else {<a name="line.1851"></a>
-<span class="sourceLineNo">1852</span> LOG.trace("Store update {}", procedure);<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span> if (procedure.isFinished() && !procedure.hasParent()) {<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span> // remove child procedures<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span> final long[] childProcIds = procStack.getSubprocedureIds();<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span> if (childProcIds != null) {<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span> store.delete(procedure, childProcIds);<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span> for (int i = 0; i < childProcIds.length; ++i) {<a name="line.1858"></a>
-<span class="sourceLineNo">1859</span> procedures.remove(childProcIds[i]);<a name="line.1859"></a>
-<span class="sourceLineNo">1860</span> }<a name="line.1860"></a>
-<span class="sourceLineNo">1861</span> } else {<a name="line.1861"></a>
-<span class="sourceLineNo">1862</span> store.update(procedure);<a name="line.1862"></a>
-<span class="sourceLineNo">1863</span> }<a name="line.1863"></a>
-<span class="sourceLineNo">1864</span> } else {<a name="line.1864"></a>
-<span class="sourceLineNo">1865</span> store.update(procedure);<a name="line.1865"></a>
-<span class="sourceLineNo">1866</span> }<a name="line.1866"></a>
-<span class="sourceLineNo">1867</span> }<a name="line.1867"></a>
-<span class="sourceLineNo">1868</span> }<a name="line.1868"></a>
-<span class="sourceLineNo">1869</span><a name="line.1869"></a>
-<span class="sourceLineNo">1870</span> private void handleInterruptedException(Procedure<TEnvironment> proc, InterruptedException e) {<a name="line.1870"></a>
-<span class="sourceLineNo">1871</span> LOG.trace("Interrupt during {}. suspend and retry it later.", proc, e);<a name="line.1871"></a>
-<span class="sourceLineNo">1872</span> // NOTE: We don't call Thread.currentThread().interrupt()<a name="line.1872"></a>
-<span class="sourceLineNo">1873</span> // because otherwise all the subsequent calls e.g. Thread.sleep() will throw<a name="line.1873"></a>
-<span class="sourceLineNo">1874</span> // the InterruptedException. If the master is going down, we will be notified<a name="line.1874"></a>
-<span class="sourceLineNo">1875</span> // and the executor/store will be stopped.<a name="line.1875"></a>
-<span class="sourceLineNo">1876</span> // (The interrupted procedure will be retried on the next run)<a name="line.1876"></a>
-<span class="sourceLineNo">1877</span> }<a name="line.1877"></a>
-<span class="sourceLineNo">1878</span><a name="line.1878"></a>
-<span class="sourceLineNo">1879</span> private void execCompletionCleanup(Procedure<TEnvironment> proc) {<a name="line.1879"></a>
-<span class="sourceLineNo">1880</span> final TEnvironment env = getEnvironment();<a name="line.1880"></a>
-<span class="sourceLineNo">1881</span> if (proc.hasLock()) {<a name="line.1881"></a>
-<span class="sourceLineNo">1882</span> LOG.warn("Usually this should not happen, we will release the lock before if the procedure" +<a name="line.1882"></a>
-<span class="sourceLineNo">1883</span> " is finished, even if the holdLock is true, arrive here means we have some holes where" +<a name="line.1883"></a>
-<span class="sourceLineNo">1884</span> " we do not release the lock. And the releaseLock below may fail since the procedure may" +<a name="line.1884"></a>
-<span class="sourceLineNo">1885</span> " have already been deleted from the procedure store.");<a name="line.1885"></a>
-<span class="sourceLineNo">1886</span> releaseLock(proc, true);<a name="line.1886"></a>
-<span class="sourceLineNo">1887</span> }<a name="line.1887"></a>
-<span class="sourceLineNo">1888</span> try {<a name="line.1888"></a>
-<span class="sourceLineNo">1889</span> proc.completionCleanup(env);<a name="line.1889"></a>
-<span class="sourceLineNo">1890</span> } catch (Throwable e) {<a name="line.1890"></a>
-<span class="sourceLineNo">1891</span> // Catch NullPointerExceptions or similar errors...<a name="line.1891"></a>
-<span class="sourceLineNo">1892</span> LOG.error("CODE-BUG: uncatched runtime exception for procedure: " + proc, e);<a name="line.1892"></a>
-<span class="sourceLineNo">1893</span> }<a name="line.1893"></a>
-<span class="sourceLineNo">1894</span> }<a name="line.1894"></a>
-<span class="sourceLineNo">1895</span><a name="line.1895"></a>
-<span class="sourceLineNo">1896</span> private void procedureFinished(Procedure<TEnvironment> proc) {<a name="line.1896"></a>
-<span class="sourceLineNo">1897</span> // call the procedure completion cleanup handler<a name="line.1897"></a>
-<span class="sourceLineNo">1898</span> execCompletionCleanup(proc);<a name="line.1898"></a>
-<span class="sourceLineNo">1899</span><a name="line.1899"></a>
-<span class="sourceLineNo">1900</span> CompletedProcedureRetainer<TEnvironment> retainer = new CompletedProcedureRetainer<>(proc);<a name="line.1900"></a>
-<span class="sourceLineNo">1901</span><a name="line.1901"></a>
-<span class="sourceLineNo">1902</span> // update the executor internal state maps<a name="line.1902"></a>
-<span class="sourceLineNo">1903</span> if (!proc.shouldWaitClientAck(getEnvironment())) {<a name="line.1903"></a>
-<span class="sourceLineNo">1904</span> retainer.setClientAckTime(0);<a name="line.1904"></a>
-<span class="sourceLineNo">1905</span> }<a name="line.1905"></a>
+<span class="sourceLineNo">1799</span> assert subproc.getState() == ProcedureState.INITIALIZING : subproc;<a name="line.1799"></a>
+<span class="sourceLineNo">1800</span> subproc.setParentProcId(procedure.getProcId());<a name="line.1800"></a>
+<span class="sourceLineNo">1801</span> subproc.setRootProcId(rootProcId);<a name="line.1801"></a>
+<span class="sourceLineNo">1802</span> subproc.setProcId(nextProcId());<a name="line.1802"></a>
+<span class="sourceLineNo">1803</span> procStack.addSubProcedure(subproc);<a name="line.1803"></a>
+<span class="sourceLineNo">1804</span> }<a name="line.1804"></a>
+<span class="sourceLineNo">1805</span><a name="line.1805"></a>
+<span class="sourceLineNo">1806</span> if (!procedure.isFailed()) {<a name="line.1806"></a>
+<span class="sourceLineNo">1807</span> procedure.setChildrenLatch(subprocs.length);<a name="line.1807"></a>
+<span class="sourceLineNo">1808</span> switch (procedure.getState()) {<a name="line.1808"></a>
+<span class="sourceLineNo">1809</span> case RUNNABLE:<a name="line.1809"></a>
+<span class="sourceLineNo">1810</span> procedure.setState(ProcedureState.WAITING);<a name="line.1810"></a>
+<span class="sourceLineNo">1811</span> break;<a name="line.1811"></a>
+<span class="sourceLineNo">1812</span> case WAITING_TIMEOUT:<a name="line.1812"></a>
+<span class="sourceLineNo">1813</span> timeoutExecutor.add(procedure);<a name="line.1813"></a>
+<span class="sourceLineNo">1814</span> break;<a name="line.1814"></a>
+<span class="sourceLineNo">1815</span> default:<a name="line.1815"></a>
+<span class="sourceLineNo">1816</span> break;<a name="line.1816"></a>
+<span class="sourceLineNo">1817</span> }<a name="line.1817"></a>
+<span class="sourceLineNo">1818</span> }<a name="line.1818"></a>
+<span class="sourceLineNo">1819</span> return subprocs;<a name="line.1819"></a>
+<span class="sourceLineNo">1820</span> }<a name="line.1820"></a>
+<span class="sourceLineNo">1821</span><a name="line.1821"></a>
+<span class="sourceLineNo">1822</span> private void submitChildrenProcedures(Procedure<TEnvironment>[] subprocs) {<a name="line.1822"></a>
+<span class="sourceLineNo">1823</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1823"></a>
+<span class="sourceLineNo">1824</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1824"></a>
+<span class="sourceLineNo">1825</span> subproc.updateMetricsOnSubmit(getEnvironment());<a name="line.1825"></a>
+<span class="sourceLineNo">1826</span> assert !procedures.containsKey(subproc.getProcId());<a name="line.1826"></a>
+<span class="sourceLineNo">1827</span> procedures.put(subproc.getProcId(), subproc);<a name="line.1827"></a>
+<span class="sourceLineNo">1828</span> scheduler.addFront(subproc);<a name="line.1828"></a>
+<span class="sourceLineNo">1829</span> }<a name="line.1829"></a>
+<span class="sourceLineNo">1830</span> }<a name="line.1830"></a>
+<span class="sourceLineNo">1831</span><a name="line.1831"></a>
+<span class="sourceLineNo">1832</span> private void countDownChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1832"></a>
+<span class="sourceLineNo">1833</span> Procedure<TEnvironment> procedure) {<a name="line.1833"></a>
+<span class="sourceLineNo">1834</span> Procedure<TEnvironment> parent = procedures.get(procedure.getParentProcId());<a name="line.1834"></a>
+<span class="sourceLineNo">1835</span> if (parent == null) {<a name="line.1835"></a>
+<span class="sourceLineNo">1836</span> assert procStack.isRollingback();<a name="line.1836"></a>
+<span class="sourceLineNo">1837</span> return;<a name="line.1837"></a>
+<span class="sourceLineNo">1838</span> }<a name="line.1838"></a>
+<span class="sourceLineNo">1839</span><a name="line.1839"></a>
+<span class="sourceLineNo">1840</span> // If this procedure is the last child awake the parent procedure<a name="line.1840"></a>
+<span class="sourceLineNo">1841</span> if (parent.tryRunnable()) {<a name="line.1841"></a>
+<span class="sourceLineNo">1842</span> // If we succeeded in making the parent runnable -- i.e. all of its<a name="line.1842"></a>
+<span class="sourceLineNo">1843</span> // children have completed, move parent to front of the queue.<a name="line.1843"></a>
+<span class="sourceLineNo">1844</span> store.update(parent);<a name="line.1844"></a>
+<span class="sourceLineNo">1845</span> scheduler.addFront(parent);<a name="line.1845"></a>
+<span class="sourceLineNo">1846</span> LOG.info("Finished subprocedure(s) of " + parent + "; resume parent processing.");<a name="line.1846"></a>
+<span class="sourceLineNo">1847</span> return;<a name="line.1847"></a>
+<span class="sourceLineNo">1848</span> }<a name="line.1848"></a>
+<span class="sourceLineNo">1849</span> }<a name="line.1849"></a>
+<span class="sourceLineNo">1850</span><a name="line.1850"></a>
+<span class="sourceLineNo">1851</span> private void updateStoreOnExec(RootProcedureState<TEnvironment> procStack,<a name="line.1851"></a>
+<span class="sourceLineNo">1852</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1852"></a>
+<span class="sourceLineNo">1853</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1853"></a>
+<span class="sourceLineNo">1854</span> if (LOG.isTraceEnabled()) {<a name="line.1854"></a>
+<span class="sourceLineNo">1855</span> LOG.trace("Stored " + procedure + ", children " + Arrays.toString(subprocs));<a name="line.1855"></a>
+<span class="sourceLineNo">1856</span> }<a name="line.1856"></a>
+<span class="sourceLineNo">1857</span> store.insert(procedure, subprocs);<a name="line.1857"></a>
+<span class="sourceLineNo">1858</span> } else {<a name="line.1858"></a>
+<span class="sourceLineNo">1859</span> LOG.trace("Store update {}", procedure);<a name="line.1859"></a>
+<span class="sourceLineNo">1860</span> if (procedure.isFinished() && !procedure.hasParent()) {<a name="line.1860"></a>
+<span class="sourceLineNo">1861</span> // remove child procedures<a name="line.1861"></a>
+<span class="sourceLineNo">1862</span> final long[] childProcIds = procStack.getSubprocedureIds();<a name="line.1862"></a>
+<span class="sourceLineNo">1863</span> if (childProcIds != null) {<a name="line.1863"></a>
+<span class="sourceLineNo">1864</span> store.delete(procedure, childProcIds);<a name="line.1864"></a>
+<span class="sourceLineNo">1865</span> for (int i = 0; i < childProcIds.length; ++i) {<a name="line.1865"></a>
+<span class="sourceLineNo">1866</span> procedures.remove(childProcIds[i]);<a name="line.1866"></a>
+<span class="sourceLineNo">1867</span> }<a name="line.1867"></a>
+<span class="sourceLineNo">1868</span> } else {<a name="line.1868"></a>
+<span class="sourceLineNo">1869</span> store.update(procedure);<a name="line.1869"></a>
+<span class="sourceLineNo">1870</span> }<a name="line.1870"></a>
+<span class="sourceLineNo">1871</span> } else {<a name="line.1871"></a>
+<span class="sourceLineNo">1872</span> store.update(procedure);<a name="line.1872"></a>
+<span class="sourceLineNo">1873</span> }<a name="line.1873"></a>
+<span class="sourceLineNo">1874</span> }<a name="line.1874"></a>
+<span class="sourceLineNo">1875</span> }<a name="line.1875"></a>
+<span class="sourceLineNo">1876</span><a name="line.1876"></a>
+<span class="sourceLineNo">1877</span> private void handleInterruptedException(Procedure<TEnvironment> proc, InterruptedException e) {<a name="line.1877"></a>
+<span class="sourceLineNo">1878</span> LOG.trace("Interrupt during {}. suspend and retry it later.", proc, e);<a name="line.1878"></a>
+<span class="sourceLineNo">1879</span> // NOTE: We don't call Thread.currentThread().interrupt()<a name="line.1879"></a>
+<span class="sourceLineNo">1880</span> // because otherwise all the subsequent calls e.g. Thread.sleep() will throw<a name="line.1880"></a>
+<span class="sourceLineNo">1881</span> // the InterruptedException. If the master is going down, we will be notified<a name="line.1881"></a>
+<span class="sourceLineNo">1882</span> // and the executor/store will be stopped.<a name="line.1882"></a>
+<span class="sourceLineNo">1883</span> // (The interrupted procedure will be retried on the next run)<a name="line.1883"></a>
+<span class="sourceLineNo">1884</span> }<a name="line.1884"></a>
+<span class="sourceLineNo">1885</span><a name="line.1885"></a>
+<span class="sourceLineNo">1886</span> private void execCompletionCleanup(Procedure<TEnvironment> proc) {<a name="line.1886"></a>
+<span class="sourceLineNo">1887</span> final TEnvironment env = getEnvironment();<a name="line.1887"></a>
+<span class="sourceLineNo">1888</span> if (proc.hasLock()) {<a name="line.1888"></a>
+<span class="sourceLineNo">1889</span> LOG.warn("Usually this should not happen, we will release the lock before if the procedure" +<a name="line.1889"></a>
+<span class="sourceLineNo">1890</span> " is finished, even if the holdLock is true, arrive here means we have some holes where" +<a name="line.1890"></a>
+<span class="sourceLineNo">1891</span> " we do not release the lock. And the releaseLock below may fail since the procedure may" +<a name="line.1891"></a>
+<span class="sourceLineNo">1892</span> " have already been deleted from the procedure store.");<a name="line.1892"></a>
+<span class="sourceLineNo">1893</span> releaseLock(proc, true);<a name="line.1893"></a>
+<span class="sourceLineNo">1894</span> }<a name="line.1894"></a>
+<span class="sourceLineNo">1895</span> try {<a name="line.1895"></a>
+<span class="sourceLineNo">1896</span> proc.completionCleanup(env);<a name="line.1896"></a>
+<span class="sourceLineNo">1897</span> } catch (Throwable e) {<a name="line.1897"></a>
+<span class="sourceLineNo">1898</span> // Catch NullPointerExceptions or similar errors...<a name="line.1898"></a>
+<span class="sourceLineNo">1899</span> LOG.error("CODE-BUG: uncatched runtime exception for procedure: " + proc, e);<a name="line.1899"></a>
+<span class="sourceLineNo">1900</span> }<a name="line.1900"></a>
+<span class="sourceLineNo">1901</span> }<a name="line.1901"></a>
+<span class="sourceLineNo">1902</span><a name="line.1902"></a>
+<span class="sourceLineNo">1903</span> private void procedureFinished(Procedure<TEnvironment> proc) {<a name="line.1903"></a>
+<span class="sourceLineNo">1904</span> // call the procedure completion cleanup handler<a name="line.1904"></a>
+<span class="sourceLineNo">1905</span> execCompletionCleanup(proc);<a name="line.1905"></a>
<span class="sourceLineNo">1906</span><a name="line.1906"></a>
-<span class="sourceLineNo">1907</span> completed.put(proc.getProcId(), retainer);<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span> rollbackStack.remove(proc.getProcId());<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span> procedures.remove(proc.getProcId());<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span><a name="line.1910"></a>
-<span class="sourceLineNo">1911</span> // call the runnableSet completion cleanup handler<a name="line.1911"></a>
-<span class="sourceLineNo">1912</span> try {<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span> scheduler.completionCleanup(proc);<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span> } catch (Throwable e) {<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span> // Catch NullPointerExceptions or similar errors...<a name="line.1915"></a>
-<span class="sourceLineNo">1916</span> LOG.error("CODE-BUG: uncatched runtime exception for completion cleanup: {}", proc, e);<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span> }<a name="line.1917"></a>
-<span class="sourceLineNo">1918</span><a name="line.1918"></a>
-<span class="sourceLineNo">1919</span> // Notify the listeners<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span> sendProcedureFinishedNotification(proc.getProcId());<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span> }<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span><a name="line.1922"></a>
-<span class="sourceLineNo">1923</span> RootProcedureState<TEnvironment> getProcStack(long rootProcId) {<a name="line.1923"></a>
-<span class="sourceLineNo">1924</span> return rollbackStack.get(rootProcId);<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span> }<a name="line.1925"></a>
-<span class="sourceLineNo">1926</span><a name="line.1926"></a>
-<span class="sourceLineNo">1927</span> @VisibleForTesting<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span> ProcedureScheduler getProcedureScheduler() {<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span> return scheduler;<a name="line.1929"></a>
-<span class="sourceLineNo">1930</span> }<a name="line.1930"></a>
-<span class="sourceLineNo">1931</span><a name="line.1931"></a>
-<span class="sourceLineNo">1932</span> @VisibleForTesting<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span> int getCompletedSize() {<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span> return completed.size();<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span> }<a name="line.1935"></a>
-<span class="sourceLineNo">1936</span><a name="line.1936"></a>
-<span class="sourceLineNo">1937</span> @VisibleForTesting<a name="line.1937"></a>
-<span class="sourceLineNo">1938</span> public IdLock getProcExecutionLock() {<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span> return procExecutionLock;<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span> }<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span><a name="line.1941"></a>
-<span class="sourceLineNo">1942</span> // ==========================================================================<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span> // Worker Thread<a name="line.1943"></a>
-<span class="sourceLineNo">1944</span> // ==========================================================================<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span> private class WorkerThread extends StoppableThread {<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span> private final AtomicLong executionStartTime = new AtomicLong(Long.MAX_VALUE);<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span> private volatile Procedure<TEnvironment> activeProcedure;<a name="line.1947"></a>
+<span class="sourceLineNo">1907</span> CompletedProcedureRetainer<TEnvironment> retainer = new CompletedProcedureRetainer<>(proc);<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span><a name="line.1908"></a>
+<span class="sourceLineNo">1909</span> // update the executor internal state maps<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span> if (!proc.shouldWaitClientAck(getEnvironment())) {<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span> retainer.setClientAckTime(0);<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span> }<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span><a name="line.1913"></a>
+<span class="sourceLineNo">1914</span> completed.put(proc.getProcId(), retainer);<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span> rollbackStack.remove(proc.getProcId());<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span> procedures.remove(proc.getProcId());<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span><a name="line.1917"></a>
+<span class="sourceLineNo">1918</span> // call the runnableSet completion cleanup handler<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span> try {<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span> scheduler.completionCleanup(proc);<a name="line.1920"></a>
+<span class="sourceLineNo">1921</span> } catch (Throwable e) {<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span> // Catch NullPointerExceptions or similar errors...<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span> LOG.error("CODE-BUG: uncatched runtime exception for completion cleanup: {}", proc, e);<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span> }<a name="line.1924"></a>
+<span class="sourceLineNo">1925</span><a name="line.1925"></a>
+<span class="sourceLineNo">1926</span> // Notify the listeners<a name="line.1926"></a>
+<span class="sourceLineNo">1927</span> sendProcedureFinishedNotification(proc.getProcId());<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span> }<a name="line.1928"></a>
+<span class="sourceLineNo">1929</span><a name="line.1929"></a>
+<span class="sourceLineNo">1930</span> RootProcedureState<TEnvironment> getProcStack(long rootProcId) {<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span> return rollbackStack.get(rootProcId);<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span> }<a name="line.1932"></a>
+<span class="sourceLineNo">1933</span><a name="line.1933"></a>
+<span class="sourceLineNo">1934</span> @VisibleForTesting<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span> ProcedureScheduler getProcedureScheduler() {<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span> return scheduler;<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span> }<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span><a name="line.1938"></a>
+<span class="sourceLineNo">1939</span> @VisibleForTesting<a name="line.1939"></a>
+<span class="sourceLineNo">1940</span> int getCompletedSize() {<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span> return completed.size();<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span> }<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span><a name="line.1943"></a>
+<span class="sourceLineNo">1944</span> @VisibleForTesting<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span> public IdLock getProcExecutionLock() {<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span> return procExecutionLock;<a name="line.1946"></a>
+<span class="sourceLineNo">1947</span> }<a name="line.1947"></a>
<span class="sourceLineNo">1948</span><a name="line.1948"></a>
-<span class="sourceLineNo">1949</span> public WorkerThread(ThreadGroup group) {<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span> this(group, "PEWorker-");<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span> }<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span><a name="line.1952"></a>
-<span class="sourceLineNo">1953</span> protected WorkerThread(ThreadGroup group, String prefix) {<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span> super(group, prefix + workerId.incrementAndGet());<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span> setDaemon(true);<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span> }<a name="line.1956"></a>
-<span class="sourceLineNo">1957</span><a name="line.1957"></a>
-<span class="sourceLineNo">1958</span> @Override<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span> public void sendStopSignal() {<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span> scheduler.signalAll();<a name="line.1960"></a>
-<span class="sourceLineNo">1961</span> }<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span> @Override<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span> public void run() {<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span> long lastUpdate = EnvironmentEdgeManager.currentTime();<a name="line.1964"></a>
-<span class="sourceLineNo">1965</span> try {<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span> while (isRunning() && keepAlive(lastUpdate)) {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span> @SuppressWarnings("unchecked")<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span> Procedure<TEnvironment> proc = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span> if (proc == null) {<a name="line.1969"></a>
-<span class="sourceLineNo">1970</span> continue;<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span> }<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span> this.activeProcedure = proc;<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span> int activeCount = activeExecutorCount.incrementAndGet();<a name="line.1973"></a>
-<span class="sourceLineNo">1974</span> int runningCount = store.setRunningProcedureCount(activeCount);<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span> LOG.trace("Execute pid={} runningCount={}, activeCount={}", proc.getProcId(),<a name="line.1975"></a>
-<span class="sourceLineNo">1976</span> runningCount, activeCount);<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span> executionStartTime.set(EnvironmentEdgeManager.currentTime());<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span> IdLock.Entry lockEntry = procExecutionLock.getLockEntry(proc.getProcId());<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span> try {<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span> executeProcedure(proc);<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span> } catch (AssertionError e) {<a name="line.1981"></a>
-<span class="sourceLineNo">1982</span> LOG.info("ASSERT pid=" + proc.getProcId(), e);<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span> throw e;<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span> } finally {<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1985"></a>
-<span class="sourceLineNo">1986</span> activeCount = activeExecutorCount.decrementAndGet();<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span> runningCount = store.setRunningProcedureCount(activeCount);<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span> LOG.trace("Halt pid={} runningCount={}, activeCount={}", proc.getProcId(),<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span> runningCount, activeCount);<a name="line.1989"></a>
-<span class="sourceLineNo">1990</span> this.activeProcedure = null;<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span> lastUpdate = EnvironmentEdgeManager.currentTime();<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span> executionStartTime.set(Long.MAX_VALUE);<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span> }<a name="line.1993"></a>
-<span class="sourceLineNo">1994</span> }<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span> } catch (Throwable t) {<a name="line.1995"></a>
-<span class="sourceLineNo">1996</span> LOG.warn("Worker terminating UNNATURALLY {}", this.activeProcedure, t);<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span> } finally {<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span> LOG.trace("Worker terminated.");<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span> }<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span> workerThreads.remove(this);<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span> }<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span><a name="line.2002"></a>
-<span class="sourceLineNo">2003</span> @Override<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span> public String toString() {<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span> Procedure<?> p = this.activeProcedure;<a name="line.2005"></a>
-<span class="sourceLineNo">2006</span> return getName() + "(pid=" + (p == null? Procedure.NO_PROC_ID: p.getProcId() + ")");<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span> }<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span><a name="line.2008"></a>
-<span class="sourceLineNo">2009</span> /**<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span> * @return the time since the current procedure is running<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span> */<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span> public long getCurrentRunTime() {<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span> return EnvironmentEdgeManager.currentTime() - executionStartTime.get();<a name="line.2013"></a>
+<span class="sourceLineNo">1949</span> // ==========================================================================<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span> // Worker Thread<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span> // ==========================================================================<a name="line.1951"></a>
+<span class="sourceLineNo">1952</span> private class WorkerThread extends StoppableThread {<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span> private final AtomicLong executionStartTime = new AtomicLong(Long.MAX_VALUE);<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span> private volatile Procedure<TEnvironment> activeProcedure;<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span><a name="line.1955"></a>
+<span class="sourceLineNo">1956</span> public WorkerThread(ThreadGroup group) {<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span> this(group, "PEWorker-");<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span> }<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span><a name="line.1959"></a>
+<span class="sourceLineNo">1960</span> protected WorkerThread(ThreadGroup group, String prefix) {<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span> super(group, prefix + workerId.incrementAndGet());<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span> setDaemon(true);<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span> }<a name="line.1963"></a>
+<span class="sourceLineNo">1964</span><a name="line.1964"></a>
+<span class="sourceLineNo">1965</span> @Override<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span> public void sendStopSignal() {<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span> scheduler.signalAll();<a name="line.1967"></a>
+<span class="sourceLineNo">1968</span> }<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span> @Override<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span> public void run() {<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span> long lastUpdate = EnvironmentEdgeManager.currentTime();<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span> try {<a name="line.1972"></a>
+<span class="sourceLineNo">1973</span> while (isRunning() && keepAlive(lastUpdate)) {<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span> @SuppressWarnings("unchecked")<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span> Procedure<TEnvironment> proc = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span> if (proc == null) {<a name="line.1976"></a>
+<span class="sourceLineNo">1977</span> continue;<a name="line.1977"></a>
+<span class="sourceLineNo">1978</span> }<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span> this.activeProcedure = proc;<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span> int activeCount = activeExecutorCount.incrementAndGet();<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span> int runningCount = store.setRunningProcedureCount(activeCount);<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span> LOG.trace("Execute pid={} runningCount={}, activeCount={}", proc.getProcId(),<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span> runningCount, activeCount);<a name="line.1983"></a>
+<span class="sourceLineNo">1984</span> executionStartTime.set(EnvironmentEdgeManager.currentTime());<a name="line.1984"></a>
+<span class="sourceLineNo">1985</span> IdLock.Entry lockEntry = procExecutionLock.getLockEntry(proc.getProcId());<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span> try {<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span> executeProcedure(proc);<a name="line.1987"></a>
+<span class="sourceLineNo">1988</span> } catch (AssertionError e) {<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span> LOG.info("ASSERT pid=" + proc.getProcId(), e);<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span> throw e;<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span> } finally {<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1992"></a>
+<span class="sourceLineNo">1993</span> activeCount = activeExecutorCount.decrementAndGet();<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span> runningCount = store.setRunningProcedureCount(activeCount);<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span> LOG.trace("Halt pid={} runningCount={}, activeCount={}", proc.getProcId(),<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span> runningCount, activeCount);<a name="line.1996"></a>
+<span class="sourceLineNo">1997</span> this.activeProcedure = null;<a name="line.1997"></a>
+<span class="sourceLineNo">1998</span> lastUpdate = EnvironmentEdgeManager.currentTime();<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span> executionStartTime.set(Long.MAX_VALUE);<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span> }<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span> }<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span> } catch (Throwable t) {<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span> LOG.warn("Worker terminating UNNATURALLY {}", this.activeProcedure, t);<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span> } finally {<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span> LOG.trace("Worker terminated.");<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span> }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span> workerThreads.remove(this);<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span> }<a name="line.2008"></a>
+<span class="sourceLineNo">2009</span><a name="line.2009"></a>
+<span class="sourceLineNo">2010</span> @Override<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span> public String toString() {<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span> Procedure<?> p = this.activeProcedure;<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span> return getName() + "(pid=" + (p == null? Procedure.NO_PROC_ID: p.getProcId() + ")");<a name="line.2013"></a>
<span class="sourceLineNo">2014</span> }<a name="line.2014"></a>
<span class="sourceLineNo">2015</span><a name="line.2015"></a>
-<span class="sourceLineNo">2016</span> // core worker never timeout<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span> protected boolean keepAlive(long lastUpdate) {<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span> return true;<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span> }<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span> }<a name="line.2020"></a>
-<span class="sourceLineNo">2021</span><a name="line.2021"></a>
-<span class="sourceLineNo">2022</span> // A worker thread which can be added when core workers are stuck. Will timeout after<a name="line.2022"></a>
-<span class="sourceLineNo">2023</span> // keepAliveTime if there is no procedure to run.<a name="line.2023"></a>
-<span class="sourceLineNo">2024</span> private final class KeepAliveWorkerThread extends WorkerThread {<a name="line.2024"></a>
-<span class="sourceLineNo">2025</span> public KeepAliveWorkerThread(ThreadGroup group) {<a name="line.2025"></a>
-<span class="sourceLineNo">2026</span> super(group, "KeepAlivePEWorker-");<a name="line.2026"></a>
-<span class="sourceLineNo">2027</span> }<a name="line.2027"></a>
+<span class="sourceLineNo">2016</span> /**<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span> * @return the time since the current procedure is running<a name="line.2017"></a>
+<span class="sourceLineNo">2018</span> */<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span> public long getCurrentRunTime() {<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span> return EnvironmentEdgeManager.currentTime() - executionStartTime.get();<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span> }<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span><a name="line.2022"></a>
+<span class="sourceLineNo">2023</span> // core worker never timeout<a name="line.2023"></a>
+<span class="sourceLineNo">2024</span> protected boolean keepAlive(long lastUpdate) {<a name="line.2024"></a>
+<span class="sourceLineNo">2025</span> return true;<a name="line.2025"></a>
+<span class="sourceLineNo">2026</span> }<a name="line.2026"></a>
+<span class="sourceLineNo">2027</span> }<a name="line.2027"></a>
<span class="sourceLineNo">2028</span><a name="line.2028"></a>
-<span class="sourceLineNo">2029</span> @Override<a name="line.2029"></a>
-<span class="sourceLineNo">2030</span> protected boolean keepAlive(long lastUpdate) {<a name="line.2030"></a>
-<span class="sourceLineNo">2031</span> return EnvironmentEdgeManager.currentTime() - lastUpdate < keepAliveTime;<a name="line.2031"></a>
-<span class="sourceLineNo">2032</span> }<a name="line.2032"></a>
-<span class="sourceLineNo">2033</span> }<a name="line.2033"></a>
-<span class="sourceLineNo">2034</span><a name="line.2034"></a>
-<span class="sourceLineNo">2035</span> // ----------------------------------------------------------------------------<a name="line.2035"></a>
-<span class="sourceLineNo">2036</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.2036"></a>
-<span class="sourceLineNo">2037</span> // full set of procedures pending and completed to write a compacted<a name="line.2037"></a>
-<span class="sourceLineNo">2038</span> // version of the log (in case is a log)?<a name="line.2038"></a>
-<span class="sourceLineNo">2039</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.2039"></a>
-<span class="sourceLineNo">2040</span> // will have the tracker saying everything is in the last log.<a name="line.2040"></a>
-<span class="sourceLineNo">2041</span> // ----------------------------------------------------------------------------<a name="line.2041"></a>
-<span class="sourceLineNo">2042</span><a name="line.2042"></a>
-<span class="sourceLineNo">2043</span> private final class WorkerMonitor extends InlineChore {<a name="line.2043"></a>
-<span class="sourceLineNo">2044</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.2044"></a>
-<span class="sourceLineNo">2045</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.2045"></a>
-<span class="sourceLineNo">2046</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.2046"></a>
-<span class="sourceLineNo">2047</span><a name="line.2047"></a>
-<span class="sourceLineNo">2048</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.2048"></a>
-<span class="sourceLineNo">2049</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.2049"></a>
-<span class="sourceLineNo">2050</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.2050"></a>
-<span class="sourceLineNo">2051</span><a name="line.2051"></a>
-<span class="sourceLineNo">2052</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.2052"></a>
-<span class="sourceLineNo">2053</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.2053"></a>
-<span class="sourceLineNo">2054</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.2054"></a>
-<span class="sourceLineNo">2055</span><a name="line.2055"></a>
-<span class="sourceLineNo">2056</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.2056"></a>
-<span class="sourceLineNo">2057</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.2057"></a>
-<span class="sourceLineNo">2058</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.2058"></a>
-<span class="sourceLineNo">2059</span><a name="line.2059"></a>
-<span class="sourceLineNo">2060</span> public WorkerMonitor() {<a name="line.2060"></a>
-<span class="sourceLineNo">2061</span> refreshConfig();<a name="line.2061"></a>
-<span class="sourceLineNo">2062</span> }<a name="line.2062"></a>
-<span class="sourceLineNo">2063</span><a name="line.2063"></a>
-<span class="sourceLineNo">2064</span> @Override<a name="line.2064"></a>
-<span class="sourceLineNo">2065</span> public void run() {<a name="line.2065"></a>
-<span class="sourceLineNo">2066</span> final int stuckCount = checkForStuckWorkers();<a name="line.2066"></a>
-<span class="sourceLineNo">2067</span> checkThreadCount(stuckCount);<a name="line.2067"></a>
-<span class="sourceLineNo">2068</span><a name="line.2068"></a>
-<span class="sourceLineNo">2069</span> // refresh interval (poor man dynamic conf update)<a name="line.2069"></a>
-<span class="sourceLineNo">2070</span> refreshConfig();<a name="line.2070"></a>
-<span class="sourceLineNo">2071</span> }<a name="line.2071"></a>
-<span class="sourceLineNo">2072</span><a name="line.2072"></a>
-<span class="sourceLineNo">2073</span> private int checkForStuckWorkers() {<a name="line.2073"></a>
-<span class="sourceLineNo">2074</span> // check if any of the worker is stuck<a name="line.2074"></a>
-<span class="sourceLineNo">2075</span> int stuckCount = 0;<a name="line.2075"></a>
-<span class="sourceLineNo">2076</span> for (WorkerThread worker : workerThreads) {<a name="line.2076"></a>
-<span class="sourceLineNo">2077</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.2077"></a>
-<span class="sourceLineNo">2078</span> continue;<a name="line.2078"></a>
-<span class="sourceLineNo">2079</span> }<a name="line.2079"></a>
-<span class="sourceLineNo">2080</span><a name="line.2080"></a>
-<span class="sourceLineNo">2081</span> // WARN the worker is stuck<a name="line.2081"></a>
-<span class="sourceLineNo">2082</span> stuckCount++;<a name="line.2082"></a>
-<span class="sourceLineNo">2083</span> LOG.warn("Worker stuck {}, run time {}", worker,<a name="line.2083"></a>
-<span class="sourceLineNo">2084</span> StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.2084"></a>
-<span class="sourceLineNo">2085</span> }<a name="line.2085"></a>
-<span class="sourceLineNo">2086</span> return stuckCount;<a name="line.2086"></a>
-<span class="sourceLineNo">2087</span> }<a name="line.2087"></a>
-<span class="sourceLineNo">2088</span><a name="line.2088"></a>
-<span class="sourceLineNo">2089</span> private void checkThreadCount(final int stuckCount) {<a name="line.2089"></a>
-<span class="sourceLineNo">2090</span> // nothing to do if there are no runnable tasks<a name="line.2090"></a>
-<span class="sourceLineNo">2091</span> if (stuckCount < 1 || !scheduler.hasRunnables()) {<a name="line.2091"></a>
-<span class="sourceLineNo">2092</span> return;<a name="line.2092"></a>
-<span class="sourceLineNo">2093</span> }<a name="line.2093"></a>
-<span class="sourceLineNo">2094</span><a name="line.2094"></a>
-<span class="sourceLineNo">2095</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.2095"></a>
-<span class="sourceLineNo">2096</span> // and every handler is active.<a name="line.2096"></a>
-<span class="sourceLineNo">2097</span> final float stuckPerc = ((float) stuckCount) / workerThreads.size();<a name="line.2097"></a>
-<span class="sourceLineNo">2098</span> // let's add new worker thread more aggressively, as they will timeout finally if there is no<a name="line.2098"></a>
-<span class="sourceLineNo">2099</span> // work to do.<a name="line.2099"></a>
-<span class="sourceLineNo">2100</span> if (stuckPerc >= addWorkerStuckPercentage && workerThreads.size() < maxPoolSize) {<a name="line.2100"></a>
-<span class="sourceLineNo">2101</span> final KeepAliveWorkerThread worker = new KeepAliveWorkerThread(threadGroup);<a name="line.2101"></a>
-<span class="sourceLineNo">2102</span> workerThreads.add(worker);<a name="line.2102"></a>
-<span class="sourceLineNo">2103</span> worker.start();<a name="line.2103"></a>
-<span class="sourceLineNo">2104</span> LOG.debug("Added new worker thread {}", worker);<a name="line.2104"></a>
-<span class="sourceLineNo">2105</span> }<a name="line.2105"></a>
-<span class="sourceLineNo">2106</span> }<a name="line.2106"></a>
-<span class="sourceLineNo">2107</span><a name="line.2107"></a>
-<span class="sourceLineNo">2108</span> private void refreshConfig() {<a name="line.2108"></a>
-<span class="sourceLineNo">2109</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2109"></a>
-<span class="sourceLineNo">2110</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2110"></a>
-<span class="sourceLineNo">2111</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2111"></a>
-<span class="sourceLineNo">2112</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2112"></a>
-<span class="sourceLineNo">2113</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2113"></a>
-<span class="sourceLineNo">2114</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2114"></a>
-<span class="sourceLineNo">2115</span> }<a name="line.2115"></a>
-<span class="sourceLineNo">2116</span><a name="line.2116"></a>
-<span class="sourceLineNo">2117</span> @Override<a name="line.2117"></a>
-<span class="sourceLineNo">2118</span> public int getTimeoutInterval() {<a name="line.2118"></a>
-<span class="sourceLineNo">2119</span> return timeoutInterval;<a name="line.2119"></a>
-<span class="sourceLineNo">2120</span> }<a name="line.2120"></a>
-<span class="sourceLineNo">2121</span> }<a name="line.2121"></a>
-<span class="sourceLineNo">2122</span>}<a name="line.2122"></a>
+<span class="sourceLineNo">2029</span> // A worker thread which can be added when core workers are stuck. Will timeout after<a name="line.2029"></a>
+<span class="sourceLineNo">2030</span> // keepAliveTime if there is no procedure to run.<a name="line.2030"></a>
+<span class="sourceLineNo">2031</span> private final class KeepAliveWorkerThread extends WorkerThread {<a name="line.2031"></a>
+<span class="sourceLineNo">2032</span> public KeepAliveWorkerThread(ThreadGroup group) {<a name="line.2032"></a>
+<span class="sourceLineNo">2033</span> super(group, "KeepAlivePEWorker-");<a name="line.2033"></a>
+<span class="sourceLineNo">2034</span> }<a name="line.2034"></a>
+<span class="sourceLineNo">2035</span><a name="line.2035"></a>
+<span class="sourceLineNo">2036</span> @Override<a name="line.2036"></a>
+<span class="sourceLineNo">2037</span> protected boolean keepAlive(long lastUpdate) {<a name="line.2037"></a>
+<span class="sourceLineNo">2038</span> return EnvironmentEdgeManager.currentTime() - lastUpdate < keepAliveTime;<a name="line.2038"></a>
+<span class="sourceLineNo">2039</span> }<a name="line.2039"></a>
+<span class="sourceLineNo">2040</span> }<a name="line.2040"></a>
+<span class="sourceLineNo">2041</span><a name="line.2041"></a>
+<span class="sourceLineNo">2042</span> // ----------------------------------------------------------------------------<a name="line.2042"></a>
+<span class="sourceLineNo">2043</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.2043"></a>
+<span class="sourceLineNo">2044</span> // full set of procedures pending and completed to write a compacted<a name="line.2044"></a>
+<span class="sourceLineNo">2045</span> // version of the log (in case is a log)?<a name="line.2045"></a>
+<span class="sourceLineNo">2046</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.2046"></a>
+<span class="sourceLineNo">2047</span> // will have the tracker saying everything is in the last log.<a name="line.2047"></a>
+<span class="sourceLineNo">2048</span> // ----------------------------------------------------------------------------<a name="line.2048"></a>
+<span class="sourceLineNo">2049</span><a name="line.2049"></a>
+<span class="sourceLineNo">2050</span> private final class WorkerMonitor extends InlineChore {<a name="line.2050"></a>
+<span class="sourceLineNo">2051</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.2051"></a>
+<span class="sourceLineNo">2052</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.2052"></a>
+<span class="sourceLineNo">2053</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.2053"></a>
+<span class="sourceLineNo">2054</span><a name="line.2054"></a>
+<span class="sourceLineNo">2055</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.2055"></a>
+<span class="sourceLineNo">2056</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.2056"></a>
+<span class="sourceLineNo">2057</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.2057"></a>
+<span class="sourceLineNo">2058</span><a name="line.2058"></a>
+<span class="sourceLineNo">2059</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.2059"></a>
+<span class="sourceLineNo">2060</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.2060"></a>
+<span class="sourceLineNo">2061</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.2061"></a>
+<span class="sourceLineNo">2062</span><a name="line.2062"></a>
+<span class="sourceLineNo">2063</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.2063"></a>
+<span class="sourceLineNo">2064</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.2064"></a>
+<span class="sourceLineNo">2065</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.2065"></a>
+<span class="sourceLineNo">2066</span><a name="line.2066"></a>
+<span class="sourceLineNo">2067</span> public WorkerMonitor() {<a name="line.2067"></a>
+<span class="sourceLineNo">2068</span> refreshConfig();<a name="line.2068"></a>
+<span class="sourceLineNo">2069</span> }<a name="line.2069"></a>
+<span class="sourceLineNo">2070</span><a name="line.2070"></a>
+<span class="sourceLineNo">2071</span> @Override<a name="line.2071"></a>
+<span class="sourceLineNo">2072</span> public void run() {<a name="line.2072"></a>
+<span class="sourceLineNo">2073</span> final int stuckCount = checkForStuckWorkers();<a name="line.2073"></a>
+<span class="sourceLineNo">2074</span> checkThreadCount(stuckCount);<a name="line.2074"></a>
+<span class="sourceLineNo">2075</span><a name="line.2075"></a>
+<span class="sourceLineNo">2076</span> // refresh interval (poor man dynamic conf update)<a name="line.2076"></a>
+<span class="sourceLineNo">2077</span> refreshConfig();<a name="line.2077"></a>
+<span class="sourceLineNo">2078</span> }<a name="line.2078"></a>
+<span class="sourceLineNo">2079</span><a name="line.2079"></a>
+<span class="sourceLineNo">2080</span> private int checkForStuckWorkers() {<a name="line.2080"></a>
+<span class="sourceLineNo">2081</span> // check if any of the worker is stuck<a name="line.2081"></a>
+<span class="sourceLineNo">2082</span> int stuckCount = 0;<a name="line.2082"></a>
+<span class="sourceLineNo">2083</span> for (WorkerThread worker : workerThreads) {<a name="line.2083"></a>
+<span class="sourceLineNo">2084</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.2084"></a>
+<span class="sourceLineNo">2085</span> continue;<a name="line.2085"></a>
+<span class="sourceLineNo">2086</span> }<a name="line.2086"></a>
+<span class="sourceLineNo">2087</span><a name="line.2087"></a>
+<span class="sourceLineNo">2088</span> // WARN the worker is stuck<a name="line.2088"></a>
+<span class="sourceLineNo">2089</span> stuckCount++;<a name="line.2089"></a>
+<span class="sourceLineNo">2090</span> LOG.warn("Worker stuck {}, run time {}", worker,<a name="line.2090"></a>
+<span class="sourceLineNo">2091</span> StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.2091"></a>
+<span class="sourceLineNo">2092</span> }<a name="line.2092"></a>
+<span class="sourceLineNo">2093</span> return stuckCount;<a name="line.2093"></a>
+<span class="sourceLineNo">2094</span> }<a name="line.2094"></a>
+<span class="sourceLineNo">2095</span><a name="line.2095"></a>
+<span class="sourceLineNo">2096</span> private void checkThreadCount(final int stuckCount) {<a name="line.2096"></a>
+<span class="sourceLineNo">2097</span> // nothing to do if there are no runnable tasks<a name="line.2097"></a>
+<span class="sourceLineNo">2098</span> if (stuckCount < 1 || !scheduler.hasRunnables()) {<a name="line.2098"></a>
+<span class="sourceLineNo">2099</span> return;<a name="line.2099"></a>
+<span class="sourceLineNo">2100</span> }<a name="line.2100"></a>
+<span class="sourceLineNo">2101</span><a name="line.2101"></a>
+<span class="sourceLineNo">2102</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.2102"></a>
+<span class="sourceLineNo">2103</span> // and every handler is active.<a name="line.2103"></a>
+<span class="sourceLineNo">2104</span> final float stuckPerc = ((float) stuckCount) / workerThreads.size();<a name="line.2104"></a>
+<span class="sourceLineNo">2105</span> // let's add new worker thread more aggressively, as they will timeout finally if there is no<a name="line.2105"></a>
+<span class="sourceLineNo">2106</span> // work to do.<a name="line.2106"></a>
+<span class="sourceLineNo">2107</span> if (stuckPerc >= addWorkerStuckPercentage && workerThreads.size() < maxPoolSize) {<a name="line.2107"></a>
+<span class="sourceLineNo">2108</span> final KeepAliveWorkerThread worker = new KeepAliveWorkerThread(threadGroup);<a name="line.2108"></a>
+<span class="sourceLineNo">2109</span> workerThreads.add(worker);<a name="line.2109"></a>
+<span class="sourceLineNo">2110</span> worker.start();<a name="line.2110"></a>
+<span class="sourceLineNo">2111</span> LOG.debug("Added new worker thread {}", worker);<a name="line.2111"></a>
+<span class="sourceLineNo">2112</span> }<a name="line.2112"></a>
+<span class="sourceLineNo">2113</span> }<a name="line.2113"></a>
+<span class="sourceLineNo">2114</span><a name="line.2114"></a>
+<span class="sourceLineNo">2115</span> private void refreshConfig() {<a name="line.2115"></a>
+<span class="sourceLineNo">2116</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2116"></a>
+<span class="sourceLineNo">2117</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2117"></a>
+<span class="sourceLineNo">2118</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2118"></a>
+<span class="sourceLineNo">2119</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2119"></a>
+<span class="sourceLineNo">2120</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2120"></a>
+<span class="sourceLineNo">2121</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2121"></a>
+<span class="sourceLineNo">2122</span> }<a name="line.2122"></a>
+<span class="sourceLineNo">2123</span><a name="line.2123"></a>
+<span class="sourceLineNo">2124</span> @Override<a name="line.2124"></a>
+<span class="sourceLineNo">2125</span> public int getTimeoutInterval() {<a name="line.2125"></a>
+<span class="sourceLineNo">2126</span> return timeoutInterval;<a name="line.2126"></a>
+<span class="sourceLineNo">2127</span> }<a name="line.2127"></a>
+<span class="sourceLineNo">2128</span> }<a name="line.2128"></a>
+<span class="sourceLineNo">2129</span>}<a name="line.2129"></a>
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
index 7816bab..95831a0 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
@@ -25,2109 +25,2116 @@
<span class="sourceLineNo">017</span> */<a name="line.17"></a>
<span class="sourceLineNo">018</span>package org.apache.hadoop.hbase.procedure2;<a name="line.18"></a>
<span class="sourceLineNo">019</span><a name="line.19"></a>
-<span class="sourceLineNo">020</span>import java.io.IOException;<a name="line.20"></a>
-<span class="sourceLineNo">021</span>import java.io.UncheckedIOException;<a name="line.21"></a>
-<span class="sourceLineNo">022</span>import java.util.ArrayDeque;<a name="line.22"></a>
-<span class="sourceLineNo">023</span>import java.util.ArrayList;<a name="line.23"></a>
-<span class="sourceLineNo">024</span>import java.util.Arrays;<a name="line.24"></a>
-<span class="sourceLineNo">025</span>import java.util.Collection;<a name="line.25"></a>
-<span class="sourceLineNo">026</span>import java.util.Deque;<a name="line.26"></a>
-<span class="sourceLineNo">027</span>import java.util.HashSet;<a name="line.27"></a>
-<span class="sourceLineNo">028</span>import java.util.List;<a name="line.28"></a>
-<span class="sourceLineNo">029</span>import java.util.Set;<a name="line.29"></a>
-<span class="sourceLineNo">030</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.30"></a>
-<span class="sourceLineNo">031</span>import java.util.concurrent.CopyOnWriteArrayList;<a name="line.31"></a>
-<span class="sourceLineNo">032</span>import java.util.concurrent.Executor;<a name="line.32"></a>
-<span class="sourceLineNo">033</span>import java.util.concurrent.Executors;<a name="line.33"></a>
-<span class="sourceLineNo">034</span>import java.util.concurrent.TimeUnit;<a name="line.34"></a>
-<span class="sourceLineNo">035</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.35"></a>
-<span class="sourceLineNo">036</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.36"></a>
-<span class="sourceLineNo">037</span>import java.util.concurrent.atomic.AtomicLong;<a name="line.37"></a>
-<span class="sourceLineNo">038</span>import java.util.stream.Collectors;<a name="line.38"></a>
-<span class="sourceLineNo">039</span>import java.util.stream.Stream;<a name="line.39"></a>
-<span class="sourceLineNo">040</span>import org.apache.hadoop.conf.Configuration;<a name="line.40"></a>
-<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.HConstants;<a name="line.41"></a>
-<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;<a name="line.42"></a>
-<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.log.HBaseMarkers;<a name="line.43"></a>
-<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.procedure2.Procedure.LockState;<a name="line.44"></a>
-<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;<a name="line.45"></a>
-<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;<a name="line.46"></a>
-<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureStoreListener;<a name="line.47"></a>
-<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.procedure2.util.StringUtils;<a name="line.48"></a>
-<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.security.User;<a name="line.49"></a>
-<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.50"></a>
-<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.util.IdLock;<a name="line.51"></a>
-<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.util.NonceKey;<a name="line.52"></a>
-<span class="sourceLineNo">053</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.53"></a>
-<span class="sourceLineNo">054</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.54"></a>
-<span class="sourceLineNo">055</span>import org.slf4j.Logger;<a name="line.55"></a>
-<span class="sourceLineNo">056</span>import org.slf4j.LoggerFactory;<a name="line.56"></a>
-<span class="sourceLineNo">057</span><a name="line.57"></a>
-<span class="sourceLineNo">058</span>import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;<a name="line.58"></a>
-<span class="sourceLineNo">059</span>import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;<a name="line.59"></a>
-<span class="sourceLineNo">060</span>import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;<a name="line.60"></a>
-<span class="sourceLineNo">061</span><a name="line.61"></a>
-<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.62"></a>
-<span class="sourceLineNo">063</span><a name="line.63"></a>
-<span class="sourceLineNo">064</span>/**<a name="line.64"></a>
-<span class="sourceLineNo">065</span> * Thread Pool that executes the submitted procedures.<a name="line.65"></a>
-<span class="sourceLineNo">066</span> * The executor has a ProcedureStore associated.<a name="line.66"></a>
-<span class="sourceLineNo">067</span> * Each operation is logged and on restart the pending procedures are resumed.<a name="line.67"></a>
-<span class="sourceLineNo">068</span> *<a name="line.68"></a>
-<span class="sourceLineNo">069</span> * Unless the Procedure code throws an error (e.g. invalid user input)<a name="line.69"></a>
-<span class="sourceLineNo">070</span> * the procedure will complete (at some point in time), On restart the pending<a name="line.70"></a>
-<span class="sourceLineNo">071</span> * procedures are resumed and the once failed will be rolledback.<a name="line.71"></a>
-<span class="sourceLineNo">072</span> *<a name="line.72"></a>
-<span class="sourceLineNo">073</span> * The user can add procedures to the executor via submitProcedure(proc)<a name="line.73"></a>
-<span class="sourceLineNo">074</span> * check for the finished state via isFinished(procId)<a name="line.74"></a>
-<span class="sourceLineNo">075</span> * and get the result via getResult(procId)<a name="line.75"></a>
-<span class="sourceLineNo">076</span> */<a name="line.76"></a>
-<span class="sourceLineNo">077</span>@InterfaceAudience.Private<a name="line.77"></a>
-<span class="sourceLineNo">078</span>public class ProcedureExecutor<TEnvironment> {<a name="line.78"></a>
-<span class="sourceLineNo">079</span> private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);<a name="line.79"></a>
-<span class="sourceLineNo">080</span><a name="line.80"></a>
-<span class="sourceLineNo">081</span> public static final String CHECK_OWNER_SET_CONF_KEY = "hbase.procedure.check.owner.set";<a name="line.81"></a>
-<span class="sourceLineNo">082</span> private static final boolean DEFAULT_CHECK_OWNER_SET = false;<a name="line.82"></a>
-<span class="sourceLineNo">083</span><a name="line.83"></a>
-<span class="sourceLineNo">084</span> public static final String WORKER_KEEP_ALIVE_TIME_CONF_KEY =<a name="line.84"></a>
-<span class="sourceLineNo">085</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.85"></a>
-<span class="sourceLineNo">086</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.86"></a>
-<span class="sourceLineNo">087</span><a name="line.87"></a>
-<span class="sourceLineNo">088</span> public static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.88"></a>
-<span class="sourceLineNo">089</span> static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.89"></a>
-<span class="sourceLineNo">090</span><a name="line.90"></a>
-<span class="sourceLineNo">091</span> public static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.91"></a>
-<span class="sourceLineNo">092</span> static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.92"></a>
-<span class="sourceLineNo">093</span><a name="line.93"></a>
-<span class="sourceLineNo">094</span> /**<a name="line.94"></a>
-<span class="sourceLineNo">095</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.95"></a>
-<span class="sourceLineNo">096</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.96"></a>
-<span class="sourceLineNo">097</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.97"></a>
-<span class="sourceLineNo">098</span> */<a name="line.98"></a>
-<span class="sourceLineNo">099</span> volatile Testing testing = null;<a name="line.99"></a>
-<span class="sourceLineNo">100</span><a name="line.100"></a>
-<span class="sourceLineNo">101</span> /**<a name="line.101"></a>
-<span class="sourceLineNo">102</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.102"></a>
-<span class="sourceLineNo">103</span> */<a name="line.103"></a>
-<span class="sourceLineNo">104</span> public static class Testing {<a name="line.104"></a>
-<span class="sourceLineNo">105</span> protected volatile boolean killIfHasParent = true;<a name="line.105"></a>
-<span class="sourceLineNo">106</span> protected volatile boolean killIfSuspended = false;<a name="line.106"></a>
-<span class="sourceLineNo">107</span><a name="line.107"></a>
-<span class="sourceLineNo">108</span> /**<a name="line.108"></a>
-<span class="sourceLineNo">109</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.109"></a>
-<span class="sourceLineNo">110</span> * persisting all the state it needs to recover after a crash.<a name="line.110"></a>
-<span class="sourceLineNo">111</span> */<a name="line.111"></a>
-<span class="sourceLineNo">112</span> protected volatile boolean killBeforeStoreUpdate = false;<a name="line.112"></a>
-<span class="sourceLineNo">113</span> protected volatile boolean toggleKillBeforeStoreUpdate = false;<a name="line.113"></a>
-<span class="sourceLineNo">114</span><a name="line.114"></a>
-<span class="sourceLineNo">115</span> /**<a name="line.115"></a>
-<span class="sourceLineNo">116</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.116"></a>
-<span class="sourceLineNo">117</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.117"></a>
-<span class="sourceLineNo">118</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.118"></a>
-<span class="sourceLineNo">119</span> */<a name="line.119"></a>
-<span class="sourceLineNo">120</span> protected volatile boolean killAfterStoreUpdate = false;<a name="line.120"></a>
-<span class="sourceLineNo">121</span> protected volatile boolean toggleKillAfterStoreUpdate = false;<a name="line.121"></a>
-<span class="sourceLineNo">122</span><a name="line.122"></a>
-<span class="sourceLineNo">123</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.123"></a>
-<span class="sourceLineNo">124</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.124"></a>
-<span class="sourceLineNo">125</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.125"></a>
-<span class="sourceLineNo">126</span> this.killBeforeStoreUpdate = !kill;<a name="line.126"></a>
-<span class="sourceLineNo">127</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.127"></a>
-<span class="sourceLineNo">128</span> }<a name="line.128"></a>
-<span class="sourceLineNo">129</span> return kill;<a name="line.129"></a>
-<span class="sourceLineNo">130</span> }<a name="line.130"></a>
-<span class="sourceLineNo">131</span><a name="line.131"></a>
-<span class="sourceLineNo">132</span> protected boolean shouldKillBeforeStoreUpdate(boolean isSuspended, boolean hasParent) {<a name="line.132"></a>
-<span class="sourceLineNo">133</span> if (isSuspended && !killIfSuspended) {<a name="line.133"></a>
-<span class="sourceLineNo">134</span> return false;<a name="line.134"></a>
-<span class="sourceLineNo">135</span> }<a name="line.135"></a>
-<span class="sourceLineNo">136</span> if (hasParent && !killIfHasParent) {<a name="line.136"></a>
-<span class="sourceLineNo">137</span> return false;<a name="line.137"></a>
-<span class="sourceLineNo">138</span> }<a name="line.138"></a>
-<span class="sourceLineNo">139</span> return shouldKillBeforeStoreUpdate();<a name="line.139"></a>
-<span class="sourceLineNo">140</span> }<a name="line.140"></a>
-<span class="sourceLineNo">141</span><a name="line.141"></a>
-<span class="sourceLineNo">142</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.142"></a>
-<span class="sourceLineNo">143</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.143"></a>
-<span class="sourceLineNo">144</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.144"></a>
-<span class="sourceLineNo">145</span> this.killAfterStoreUpdate = !kill;<a name="line.145"></a>
-<span class="sourceLineNo">146</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.146"></a>
-<span class="sourceLineNo">147</span> }<a name="line.147"></a>
-<span class="sourceLineNo">148</span> return kill;<a name="line.148"></a>
-<span class="sourceLineNo">149</span> }<a name="line.149"></a>
-<span class="sourceLineNo">150</span><a name="line.150"></a>
-<span class="sourceLineNo">151</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.151"></a>
-<span class="sourceLineNo">152</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.152"></a>
-<span class="sourceLineNo">153</span> }<a name="line.153"></a>
-<span class="sourceLineNo">154</span> }<a name="line.154"></a>
-<span class="sourceLineNo">155</span><a name="line.155"></a>
-<span class="sourceLineNo">156</span> public interface ProcedureExecutorListener {<a name="line.156"></a>
-<span class="sourceLineNo">157</span> void procedureLoaded(long procId);<a name="line.157"></a>
-<span class="sourceLineNo">158</span> void procedureAdded(long procId);<a name="line.158"></a>
-<span class="sourceLineNo">159</span> void procedureFinished(long procId);<a name="line.159"></a>
-<span class="sourceLineNo">160</span> }<a name="line.160"></a>
-<span class="sourceLineNo">161</span><a name="line.161"></a>
-<span class="sourceLineNo">162</span> /**<a name="line.162"></a>
-<span class="sourceLineNo">163</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.163"></a>
-<span class="sourceLineNo">164</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.164"></a>
-<span class="sourceLineNo">165</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.165"></a>
-<span class="sourceLineNo">166</span> */<a name="line.166"></a>
-<span class="sourceLineNo">167</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.167"></a>
-<span class="sourceLineNo">168</span> new ConcurrentHashMap<>();<a name="line.168"></a>
-<span class="sourceLineNo">169</span><a name="line.169"></a>
-<span class="sourceLineNo">170</span> /**<a name="line.170"></a>
-<span class="sourceLineNo">171</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.171"></a>
-<span class="sourceLineNo">172</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.172"></a>
-<span class="sourceLineNo">173</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.173"></a>
-<span class="sourceLineNo">174</span> */<a name="line.174"></a>
-<span class="sourceLineNo">175</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.175"></a>
-<span class="sourceLineNo">176</span> new ConcurrentHashMap<>();<a name="line.176"></a>
-<span class="sourceLineNo">177</span><a name="line.177"></a>
-<span class="sourceLineNo">178</span> /**<a name="line.178"></a>
-<span class="sourceLineNo">179</span> * Helper map to lookup the live procedures by ID.<a name="line.179"></a>
-<span class="sourceLineNo">180</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.180"></a>
-<span class="sourceLineNo">181</span> */<a name="line.181"></a>
-<span class="sourceLineNo">182</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.182"></a>
-<span class="sourceLineNo">183</span> new ConcurrentHashMap<>();<a name="line.183"></a>
-<span class="sourceLineNo">184</span><a name="line.184"></a>
-<span class="sourceLineNo">185</span> /**<a name="line.185"></a>
-<span class="sourceLineNo">186</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.186"></a>
-<span class="sourceLineNo">187</span> * contains every root procedure.<a name="line.187"></a>
-<span class="sourceLineNo">188</span> */<a name="line.188"></a>
-<span class="sourceLineNo">189</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.189"></a>
-<span class="sourceLineNo">190</span><a name="line.190"></a>
-<span class="sourceLineNo">191</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.191"></a>
-<span class="sourceLineNo">192</span> new CopyOnWriteArrayList<>();<a name="line.192"></a>
-<span class="sourceLineNo">193</span><a name="line.193"></a>
-<span class="sourceLineNo">194</span> private Configuration conf;<a name="line.194"></a>
-<span class="sourceLineNo">195</span><a name="line.195"></a>
-<span class="sourceLineNo">196</span> /**<a name="line.196"></a>
-<span class="sourceLineNo">197</span> * Created in the {@link #init(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.197"></a>
-<span class="sourceLineNo">198</span> * resource handling rather than observing in a #join is unexpected).<a name="line.198"></a>
-<span class="sourceLineNo">199</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.199"></a>
-<span class="sourceLineNo">200</span> * (Should be ok).<a name="line.200"></a>
-<span class="sourceLineNo">201</span> */<a name="line.201"></a>
-<span class="sourceLineNo">202</span> private ThreadGroup threadGroup;<a name="line.202"></a>
-<span class="sourceLineNo">203</span><a name="line.203"></a>
-<span class="sourceLineNo">204</span> /**<a name="line.204"></a>
-<span class="sourceLineNo">205</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.205"></a>
-<span class="sourceLineNo">206</span> * resource handling rather than observing in a #join is unexpected).<a name="line.206"></a>
-<span class="sourceLineNo">207</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.207"></a>
-<span class="sourceLineNo">208</span> * (Should be ok).<a name="line.208"></a>
-<span class="sourceLineNo">209</span> */<a name="line.209"></a>
-<span class="sourceLineNo">210</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.210"></a>
-<span class="sourceLineNo">211</span><a name="line.211"></a>
-<span class="sourceLineNo">212</span> /**<a name="line.212"></a>
-<span class="sourceLineNo">213</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.213"></a>
-<span class="sourceLineNo">214</span> * resource handling rather than observing in a #join is unexpected).<a name="line.214"></a>
-<span class="sourceLineNo">215</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.215"></a>
-<span class="sourceLineNo">216</span> * (Should be ok).<a name="line.216"></a>
-<span class="sourceLineNo">217</span> */<a name="line.217"></a>
-<span class="sourceLineNo">218</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.218"></a>
-<span class="sourceLineNo">219</span><a name="line.219"></a>
-<span class="sourceLineNo">220</span> /**<a name="line.220"></a>
-<span class="sourceLineNo">221</span> * WorkerMonitor check for stuck workers and new worker thread when necessary, for example if<a name="line.221"></a>
-<span class="sourceLineNo">222</span> * there is no worker to assign meta, it will new worker thread for it, so it is very important.<a name="line.222"></a>
-<span class="sourceLineNo">223</span> * TimeoutExecutor execute many tasks like DeadServerMetricRegionChore RegionInTransitionChore<a name="line.223"></a>
-<span class="sourceLineNo">224</span> * and so on, some tasks may execute for a long time so will block other tasks like<a name="line.224"></a>
-<span class="sourceLineNo">225</span> * WorkerMonitor, so use a dedicated thread for executing WorkerMonitor.<a name="line.225"></a>
-<span class="sourceLineNo">226</span> */<a name="line.226"></a>
-<span class="sourceLineNo">227</span> private TimeoutExecutorThread<TEnvironment> workerMonitorExecutor;<a name="line.227"></a>
-<span class="sourceLineNo">228</span><a name="line.228"></a>
-<span class="sourceLineNo">229</span> private int corePoolSize;<a name="line.229"></a>
-<span class="sourceLineNo">230</span> private int maxPoolSize;<a name="line.230"></a>
-<span class="sourceLineNo">231</span><a name="line.231"></a>
-<span class="sourceLineNo">232</span> private volatile long keepAliveTime;<a name="line.232"></a>
-<span class="sourceLineNo">233</span><a name="line.233"></a>
-<span class="sourceLineNo">234</span> /**<a name="line.234"></a>
-<span class="sourceLineNo">235</span> * Scheduler/Queue that contains runnable procedures.<a name="line.235"></a>
-<span class="sourceLineNo">236</span> */<a name="line.236"></a>
-<span class="sourceLineNo">237</span> private final ProcedureScheduler scheduler;<a name="line.237"></a>
-<span class="sourceLineNo">238</span><a name="line.238"></a>
-<span class="sourceLineNo">239</span> private final Executor forceUpdateExecutor = Executors.newSingleThreadExecutor(<a name="line.239"></a>
-<span class="sourceLineNo">240</span> new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());<a name="line.240"></a>
-<span class="sourceLineNo">241</span><a name="line.241"></a>
-<span class="sourceLineNo">242</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.242"></a>
-<span class="sourceLineNo">243</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.243"></a>
-<span class="sourceLineNo">244</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.244"></a>
-<span class="sourceLineNo">245</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.245"></a>
-<span class="sourceLineNo">246</span> private final TEnvironment environment;<a name="line.246"></a>
-<span class="sourceLineNo">247</span> private final ProcedureStore store;<a name="line.247"></a>
-<span class="sourceLineNo">248</span><a name="line.248"></a>
-<span class="sourceLineNo">249</span> private final boolean checkOwnerSet;<a name="line.249"></a>
-<span class="sourceLineNo">250</span><a name="line.250"></a>
-<span class="sourceLineNo">251</span> // To prevent concurrent execution of the same procedure.<a name="line.251"></a>
-<span class="sourceLineNo">252</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.252"></a>
-<span class="sourceLineNo">253</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.253"></a>
-<span class="sourceLineNo">254</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.254"></a>
-<span class="sourceLineNo">255</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.255"></a>
-<span class="sourceLineNo">256</span> // execution of the same procedure.<a name="line.256"></a>
-<span class="sourceLineNo">257</span> private final IdLock procExecutionLock = new IdLock();<a name="line.257"></a>
-<span class="sourceLineNo">258</span><a name="line.258"></a>
-<span class="sourceLineNo">259</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.259"></a>
-<span class="sourceLineNo">260</span> final ProcedureStore store) {<a name="line.260"></a>
-<span class="sourceLineNo">261</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.261"></a>
-<span class="sourceLineNo">262</span> }<a name="line.262"></a>
-<span class="sourceLineNo">263</span><a name="line.263"></a>
-<span class="sourceLineNo">264</span> private boolean isRootFinished(Procedure<?> proc) {<a name="line.264"></a>
-<span class="sourceLineNo">265</span> Procedure<?> rootProc = procedures.get(proc.getRootProcId());<a name="line.265"></a>
-<span class="sourceLineNo">266</span> return rootProc == null || rootProc.isFinished();<a name="line.266"></a>
-<span class="sourceLineNo">267</span> }<a name="line.267"></a>
-<span class="sourceLineNo">268</span><a name="line.268"></a>
-<span class="sourceLineNo">269</span> private void forceUpdateProcedure(long procId) throws IOException {<a name="line.269"></a>
-<span class="sourceLineNo">270</span> IdLock.Entry lockEntry = procExecutionLock.getLockEntry(procId);<a name="line.270"></a>
-<span class="sourceLineNo">271</span> try {<a name="line.271"></a>
-<span class="sourceLineNo">272</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.272"></a>
-<span class="sourceLineNo">273</span> if (proc != null) {<a name="line.273"></a>
-<span class="sourceLineNo">274</span> if (proc.isFinished() && proc.hasParent() && isRootFinished(proc)) {<a name="line.274"></a>
-<span class="sourceLineNo">275</span> LOG.debug("Procedure {} has already been finished and parent is succeeded," +<a name="line.275"></a>
-<span class="sourceLineNo">276</span> " skip force updating", proc);<a name="line.276"></a>
-<span class="sourceLineNo">277</span> return;<a name="line.277"></a>
-<span class="sourceLineNo">278</span> }<a name="line.278"></a>
-<span class="sourceLineNo">279</span> } else {<a name="line.279"></a>
-<span class="sourceLineNo">280</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.280"></a>
-<span class="sourceLineNo">281</span> if (retainer == null || retainer.getProcedure() instanceof FailedProcedure) {<a name="line.281"></a>
-<span class="sourceLineNo">282</span> LOG.debug("No pending procedure with id = {}, skip force updating.", procId);<a name="line.282"></a>
-<span class="sourceLineNo">283</span> return;<a name="line.283"></a>
-<span class="sourceLineNo">284</span> }<a name="line.284"></a>
-<span class="sourceLineNo">285</span> long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.285"></a>
-<span class="sourceLineNo">286</span> long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.286"></a>
-<span class="sourceLineNo">287</span> if (retainer.isExpired(System.currentTimeMillis(), evictTtl, evictAckTtl)) {<a name="line.287"></a>
-<span class="sourceLineNo">288</span> LOG.debug("Procedure {} has already been finished and expired, skip force updating",<a name="line.288"></a>
-<span class="sourceLineNo">289</span> procId);<a name="line.289"></a>
-<span class="sourceLineNo">290</span> return;<a name="line.290"></a>
-<span class="sourceLineNo">291</span> }<a name="line.291"></a>
-<span class="sourceLineNo">292</span> proc = retainer.getProcedure();<a name="line.292"></a>
-<span class="sourceLineNo">293</span> }<a name="line.293"></a>
-<span class="sourceLineNo">294</span> LOG.debug("Force update procedure {}", proc);<a name="line.294"></a>
-<span class="sourceLineNo">295</span> store.update(proc);<a name="line.295"></a>
-<span class="sourceLineNo">296</span> } finally {<a name="line.296"></a>
-<span class="sourceLineNo">297</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.297"></a>
-<span class="sourceLineNo">298</span> }<a name="line.298"></a>
-<span class="sourceLineNo">299</span> }<a name="line.299"></a>
-<span class="sourceLineNo">300</span><a name="line.300"></a>
-<span class="sourceLineNo">301</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.301"></a>
-<span class="sourceLineNo">302</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.302"></a>
-<span class="sourceLineNo">303</span> this.environment = environment;<a name="line.303"></a>
-<span class="sourceLineNo">304</span> this.scheduler = scheduler;<a name="line.304"></a>
-<span class="sourceLineNo">305</span> this.store = store;<a name="line.305"></a>
-<span class="sourceLineNo">306</span> this.conf = conf;<a name="line.306"></a>
-<span class="sourceLineNo">307</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.307"></a>
-<span class="sourceLineNo">308</span> refreshConfiguration(conf);<a name="line.308"></a>
-<span class="sourceLineNo">309</span> store.registerListener(new ProcedureStoreListener() {<a name="line.309"></a>
-<span class="sourceLineNo">310</span><a name="line.310"></a>
-<span class="sourceLineNo">311</span> @Override<a name="line.311"></a>
-<span class="sourceLineNo">312</span> public void forceUpdate(long[] procIds) {<a name="line.312"></a>
-<span class="sourceLineNo">313</span> Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> {<a name="line.313"></a>
-<span class="sourceLineNo">314</span> try {<a name="line.314"></a>
-<span class="sourceLineNo">315</span> forceUpdateProcedure(procId);<a name="line.315"></a>
-<span class="sourceLineNo">316</span> } catch (IOException e) {<a name="line.316"></a>
-<span class="sourceLineNo">317</span> LOG.warn("Failed to force update procedure with pid={}", procId);<a name="line.317"></a>
-<span class="sourceLineNo">318</span> }<a name="line.318"></a>
-<span class="sourceLineNo">319</span> }));<a name="line.319"></a>
-<span class="sourceLineNo">320</span> }<a name="line.320"></a>
-<span class="sourceLineNo">321</span> });<a name="line.321"></a>
-<span class="sourceLineNo">322</span> }<a name="line.322"></a>
-<span class="sourceLineNo">323</span><a name="line.323"></a>
-<span class="sourceLineNo">324</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.324"></a>
-<span class="sourceLineNo">325</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.325"></a>
-<span class="sourceLineNo">326</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.326"></a>
-<span class="sourceLineNo">327</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.327"></a>
-<span class="sourceLineNo">328</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.328"></a>
-<span class="sourceLineNo">329</span><a name="line.329"></a>
-<span class="sourceLineNo">330</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.330"></a>
-<span class="sourceLineNo">331</span> @Override<a name="line.331"></a>
-<span class="sourceLineNo">332</span> public void setMaxProcId(long maxProcId) {<a name="line.332"></a>
-<span class="sourceLineNo">333</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.333"></a>
-<span class="sourceLineNo">334</span> lastProcId.set(maxProcId);<a name="line.334"></a>
-<span class="sourceLineNo">335</span> }<a name="line.335"></a>
-<span class="sourceLineNo">336</span><a name="line.336"></a>
-<span class="sourceLineNo">337</span> @Override<a name="line.337"></a>
-<span class="sourceLineNo">338</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.338"></a>
-<span class="sourceLineNo">339</span> loadProcedures(procIter, abortOnCorruption);<a name="line.339"></a>
-<span class="sourceLineNo">340</span> }<a name="line.340"></a>
-<span class="sourceLineNo">341</span><a name="line.341"></a>
-<span class="sourceLineNo">342</span> @Override<a name="line.342"></a>
-<span class="sourceLineNo">343</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.343"></a>
-<span class="sourceLineNo">344</span> int corruptedCount = 0;<a name="line.344"></a>
-<span class="sourceLineNo">345</span> while (procIter.hasNext()) {<a name="line.345"></a>
-<span class="sourceLineNo">346</span> Procedure<?> proc = procIter.next();<a name="line.346"></a>
-<span class="sourceLineNo">347</span> LOG.error("Corrupt " + proc);<a name="line.347"></a>
-<span class="sourceLineNo">348</span> corruptedCount++;<a name="line.348"></a>
-<span class="sourceLineNo">349</span> }<a name="line.349"></a>
-<span class="sourceLineNo">350</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.350"></a>
-<span class="sourceLineNo">351</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.351"></a>
-<span class="sourceLineNo">352</span> }<a name="line.352"></a>
-<span class="sourceLineNo">353</span> }<a name="line.353"></a>
-<span class="sourceLineNo">354</span> });<a name="line.354"></a>
-<span class="sourceLineNo">355</span> }<a name="line.355"></a>
-<span class="sourceLineNo">356</span><a name="line.356"></a>
-<span class="sourceLineNo">357</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.357"></a>
-<span class="sourceLineNo">358</span> proc.restoreLock(getEnvironment());<a name="line.358"></a>
-<span class="sourceLineNo">359</span> restored.add(proc.getProcId());<a name="line.359"></a>
-<span class="sourceLineNo">360</span> }<a name="line.360"></a>
-<span class="sourceLineNo">361</span><a name="line.361"></a>
-<span class="sourceLineNo">362</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> while (!stack.isEmpty()) {<a name="line.363"></a>
-<span class="sourceLineNo">364</span> restoreLock(stack.pop(), restored);<a name="line.364"></a>
-<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span> }<a name="line.366"></a>
-<span class="sourceLineNo">367</span><a name="line.367"></a>
-<span class="sourceLineNo">368</span> // Restore the locks for all the procedures.<a name="line.368"></a>
-<span class="sourceLineNo">369</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.369"></a>
-<span class="sourceLineNo">370</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.370"></a>
-<span class="sourceLineNo">371</span> // calling the acquireLock method for the parent procedure.<a name="line.371"></a>
-<span class="sourceLineNo">372</span> // The algorithm is straight-forward:<a name="line.372"></a>
-<span class="sourceLineNo">373</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.373"></a>
-<span class="sourceLineNo">374</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.374"></a>
-<span class="sourceLineNo">375</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.375"></a>
-<span class="sourceLineNo">376</span> // unless<a name="line.376"></a>
-<span class="sourceLineNo">377</span> // a. We have no parent, i.e, we are the root procedure<a name="line.377"></a>
-<span class="sourceLineNo">378</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.378"></a>
-<span class="sourceLineNo">379</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.379"></a>
-<span class="sourceLineNo">380</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.380"></a>
-<span class="sourceLineNo">381</span> private void restoreLocks() {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> Set<Long> restored = new HashSet<>();<a name="line.382"></a>
-<span class="sourceLineNo">383</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.383"></a>
-<span class="sourceLineNo">384</span> procedures.values().forEach(proc -> {<a name="line.384"></a>
-<span class="sourceLineNo">385</span> for (;;) {<a name="line.385"></a>
-<span class="sourceLineNo">386</span> if (restored.contains(proc.getProcId())) {<a name="line.386"></a>
-<span class="sourceLineNo">387</span> restoreLocks(stack, restored);<a name="line.387"></a>
-<span class="sourceLineNo">388</span> return;<a name="line.388"></a>
-<span class="sourceLineNo">389</span> }<a name="line.389"></a>
-<span class="sourceLineNo">390</span> if (!proc.hasParent()) {<a name="line.390"></a>
-<span class="sourceLineNo">391</span> restoreLock(proc, restored);<a name="line.391"></a>
-<span class="sourceLineNo">392</span> restoreLocks(stack, restored);<a name="line.392"></a>
-<span class="sourceLineNo">393</span> return;<a name="line.393"></a>
-<span class="sourceLineNo">394</span> }<a name="line.394"></a>
-<span class="sourceLineNo">395</span> stack.push(proc);<a name="line.395"></a>
-<span class="sourceLineNo">396</span> proc = procedures.get(proc.getParentProcId());<a name="line.396"></a>
-<span class="sourceLineNo">397</span> }<a name="line.397"></a>
-<span class="sourceLineNo">398</span> });<a name="line.398"></a>
-<span class="sourceLineNo">399</span> }<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.401"></a>
-<span class="sourceLineNo">402</span> throws IOException {<a name="line.402"></a>
-<span class="sourceLineNo">403</span> // 1. Build the rollback stack<a name="line.403"></a>
-<span class="sourceLineNo">404</span> int runnableCount = 0;<a name="line.404"></a>
-<span class="sourceLineNo">405</span> int failedCount = 0;<a name="line.405"></a>
-<span class="sourceLineNo">406</span> int waitingCount = 0;<a name="line.406"></a>
-<span class="sourceLineNo">407</span> int waitingTimeoutCount = 0;<a name="line.407"></a>
-<span class="sourceLineNo">408</span> while (procIter.hasNext()) {<a name="line.408"></a>
-<span class="sourceLineNo">409</span> boolean finished = procIter.isNextFinished();<a name="line.409"></a>
-<span class="sourceLineNo">410</span> @SuppressWarnings("unchecked")<a name="line.410"></a>
-<span class="sourceLineNo">411</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.411"></a>
-<span class="sourceLineNo">412</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.412"></a>
-<span class="sourceLineNo">413</span> long procId = proc.getProcId();<a name="line.413"></a>
-<span class="sourceLineNo">414</span><a name="line.414"></a>
-<span class="sourceLineNo">415</span> if (finished) {<a name="line.415"></a>
-<span class="sourceLineNo">416</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.416"></a>
-<span class="sourceLineNo">417</span> LOG.debug("Completed {}", proc);<a name="line.417"></a>
-<span class="sourceLineNo">418</span> } else {<a name="line.418"></a>
-<span class="sourceLineNo">419</span> if (!proc.hasParent()) {<a name="line.419"></a>
-<span class="sourceLineNo">420</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.420"></a>
-<span class="sourceLineNo">421</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.421"></a>
-<span class="sourceLineNo">422</span> }<a name="line.422"></a>
-<span class="sourceLineNo">423</span><a name="line.423"></a>
-<span class="sourceLineNo">424</span> // add the procedure to the map<a name="line.424"></a>
-<span class="sourceLineNo">425</span> proc.beforeReplay(getEnvironment());<a name="line.425"></a>
-<span class="sourceLineNo">426</span> procedures.put(proc.getProcId(), proc);<a name="line.426"></a>
-<span class="sourceLineNo">427</span> switch (proc.getState()) {<a name="line.427"></a>
-<span class="sourceLineNo">428</span> case RUNNABLE:<a name="line.428"></a>
-<span class="sourceLineNo">429</span> runnableCount++;<a name="line.429"></a>
-<span class="sourceLineNo">430</span> break;<a name="line.430"></a>
-<span class="sourceLineNo">431</span> case FAILED:<a name="line.431"></a>
-<span class="sourceLineNo">432</span> failedCount++;<a name="line.432"></a>
-<span class="sourceLineNo">433</span> break;<a name="line.433"></a>
-<span class="sourceLineNo">434</span> case WAITING:<a name="line.434"></a>
-<span class="sourceLineNo">435</span> waitingCount++;<a name="line.435"></a>
-<span class="sourceLineNo">436</span> break;<a name="line.436"></a>
-<span class="sourceLineNo">437</span> case WAITING_TIMEOUT:<a name="line.437"></a>
-<span class="sourceLineNo">438</span> waitingTimeoutCount++;<a name="line.438"></a>
-<span class="sourceLineNo">439</span> break;<a name="line.439"></a>
-<span class="sourceLineNo">440</span> default:<a name="line.440"></a>
-<span class="sourceLineNo">441</span> break;<a name="line.441"></a>
-<span class="sourceLineNo">442</span> }<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span><a name="line.444"></a>
-<span class="sourceLineNo">445</span> if (nonceKey != null) {<a name="line.445"></a>
-<span class="sourceLineNo">446</span> nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the map<a name="line.446"></a>
-<span class="sourceLineNo">447</span> }<a name="line.447"></a>
-<span class="sourceLineNo">448</span> }<a name="line.448"></a>
-<span class="sourceLineNo">449</span><a name="line.449"></a>
-<span class="sourceLineNo">450</span> // 2. Initialize the stacks: In the old implementation, for procedures in FAILED state, we will<a name="line.450"></a>
-<span class="sourceLineNo">451</span> // push it into the ProcedureScheduler directly to execute the rollback. But this does not work<a name="line.451"></a>
-<span class="sourceLineNo">452</span> // after we introduce the restore lock stage. For now, when we acquire a xlock, we will remove<a name="line.452"></a>
-<span class="sourceLineNo">453</span> // the queue from runQueue in scheduler, and then when a procedure which has lock access, for<a name="line.453"></a>
-<span class="sourceLineNo">454</span> // example, a sub procedure of the procedure which has the xlock, is pushed into the scheduler,<a name="line.454"></a>
-<span class="sourceLineNo">455</span> // we will add the queue back to let the workers poll from it. The assumption here is that, the<a name="line.455"></a>
-<span class="sourceLineNo">456</span> // procedure which has the xlock should have been polled out already, so when loading we can not<a name="line.456"></a>
-<span class="sourceLineNo">457</span> // add the procedure to scheduler first and then call acquireLock, since the procedure is still<a name="line.457"></a>
-<span class="sourceLineNo">458</span> // in the queue, and since we will remove the queue from runQueue, then no one can poll it out,<a name="line.458"></a>
-<span class="sourceLineNo">459</span> // then there is a dead lock<a name="line.459"></a>
-<span class="sourceLineNo">460</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnableCount);<a name="line.460"></a>
-<span class="sourceLineNo">461</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.461"></a>
-<span class="sourceLineNo">462</span> List<Procedure<TEnvironment>> waitingList = new ArrayList<>(waitingCount);<a name="line.462"></a>
-<span class="sourceLineNo">463</span> List<Procedure<TEnvironment>> waitingTimeoutList = new ArrayList<>(waitingTimeoutCount);<a name="line.463"></a>
-<span class="sourceLineNo">464</span> procIter.reset();<a name="line.464"></a>
-<span class="sourceLineNo">465</span> while (procIter.hasNext()) {<a name="line.465"></a>
-<span class="sourceLineNo">466</span> if (procIter.isNextFinished()) {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> procIter.skipNext();<a name="line.467"></a>
-<span class="sourceLineNo">468</span> continue;<a name="line.468"></a>
-<span class="sourceLineNo">469</span> }<a name="line.469"></a>
-<span class="sourceLineNo">470</span><a name="line.470"></a>
-<span class="sourceLineNo">471</span> @SuppressWarnings("unchecked")<a name="line.471"></a>
-<span class="sourceLineNo">472</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.472"></a>
-<span class="sourceLineNo">473</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.473"></a>
-<span class="sourceLineNo">474</span> LOG.debug("Loading {}", proc);<a name="line.474"></a>
-<span class="sourceLineNo">475</span> Long rootProcId = getRootProcedureId(proc);<a name="line.475"></a>
-<span class="sourceLineNo">476</span> // The orphan procedures will be passed to handleCorrupted, so add an assert here<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assert rootProcId != null;<a name="line.477"></a>
-<span class="sourceLineNo">478</span><a name="line.478"></a>
-<span class="sourceLineNo">479</span> if (proc.hasParent()) {<a name="line.479"></a>
-<span class="sourceLineNo">480</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.480"></a>
-<span class="sourceLineNo">481</span> if (parent != null && !proc.isFinished()) {<a name="line.481"></a>
-<span class="sourceLineNo">482</span> parent.incChildrenLatch();<a name="line.482"></a>
-<span class="sourceLineNo">483</span> }<a name="line.483"></a>
-<span class="sourceLineNo">484</span> }<a name="line.484"></a>
-<span class="sourceLineNo">485</span><a name="line.485"></a>
-<span class="sourceLineNo">486</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.486"></a>
-<span class="sourceLineNo">487</span> procStack.loadStack(proc);<a name="line.487"></a>
-<span class="sourceLineNo">488</span><a name="line.488"></a>
-<span class="sourceLineNo">489</span> proc.setRootProcId(rootProcId);<a name="line.489"></a>
-<span class="sourceLineNo">490</span> switch (proc.getState()) {<a name="line.490"></a>
-<span class="sourceLineNo">491</span> case RUNNABLE:<a name="line.491"></a>
-<span class="sourceLineNo">492</span> runnableList.add(proc);<a name="line.492"></a>
-<span class="sourceLineNo">493</span> break;<a name="line.493"></a>
-<span class="sourceLineNo">494</span> case WAITING:<a name="line.494"></a>
-<span class="sourceLineNo">495</span> waitingList.add(proc);<a name="line.495"></a>
-<span class="sourceLineNo">496</span> break;<a name="line.496"></a>
-<span class="sourceLineNo">497</span> case WAITING_TIMEOUT:<a name="line.497"></a>
-<span class="sourceLineNo">498</span> waitingTimeoutList.add(proc);<a name="line.498"></a>
-<span class="sourceLineNo">499</span> break;<a name="line.499"></a>
-<span class="sourceLineNo">500</span> case FAILED:<a name="line.500"></a>
-<span class="sourceLineNo">501</span> failedList.add(proc);<a name="line.501"></a>
-<span class="sourceLineNo">502</span> break;<a name="line.502"></a>
-<span class="sourceLineNo">503</span> case ROLLEDBACK:<a name="line.503"></a>
-<span class="sourceLineNo">504</span> case INITIALIZING:<a name="line.504"></a>
-<span class="sourceLineNo">505</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.505"></a>
-<span class="sourceLineNo">506</span> LOG.error(msg);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> throw new UnsupportedOperationException(msg);<a name="line.507"></a>
-<span class="sourceLineNo">508</span> default:<a name="line.508"></a>
-<span class="sourceLineNo">509</span> break;<a name="line.509"></a>
-<span class="sourceLineNo">510</span> }<a name="line.510"></a>
-<span class="sourceLineNo">511</span> }<a name="line.511"></a>
-<span class="sourceLineNo">512</span><a name="line.512"></a>
-<span class="sourceLineNo">513</span> // 3. Check the waiting procedures to see if some of them can be added to runnable.<a name="line.513"></a>
-<span class="sourceLineNo">514</span> waitingList.forEach(proc -> {<a name="line.514"></a>
-<span class="sourceLineNo">515</span> if (!proc.hasChildren()) {<a name="line.515"></a>
-<span class="sourceLineNo">516</span> // Normally, WAITING procedures should be waken by its children. But, there is a case that,<a name="line.516"></a>
-<span class="sourceLineNo">517</span> // all the children are successful and before they can wake up their parent procedure, the<a name="line.517"></a>
-<span class="sourceLineNo">518</span> // master was killed. So, during recovering the procedures from ProcedureWal, its children<a name="line.518"></a>
-<span class="sourceLineNo">519</span> // are not loaded because of their SUCCESS state. So we need to continue to run this WAITING<a name="line.519"></a>
-<span class="sourceLineNo">520</span> // procedure. But before executing, we need to set its state to RUNNABLE, otherwise, a<a name="line.520"></a>
-<span class="sourceLineNo">521</span> // exception will throw:<a name="line.521"></a>
-<span class="sourceLineNo">522</span> // Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.522"></a>
-<span class="sourceLineNo">523</span> // "NOT RUNNABLE! " + procedure.toString());<a name="line.523"></a>
-<span class="sourceLineNo">524</span> proc.setState(ProcedureState.RUNNABLE);<a name="line.524"></a>
-<span class="sourceLineNo">525</span> runnableList.add(proc);<a name="line.525"></a>
-<span class="sourceLineNo">526</span> } else {<a name="line.526"></a>
-<span class="sourceLineNo">527</span> proc.afterReplay(getEnvironment());<a name="line.527"></a>
-<span class="sourceLineNo">528</span> }<a name="line.528"></a>
-<span class="sourceLineNo">529</span> });<a name="line.529"></a>
-<span class="sourceLineNo">530</span> // 4. restore locks<a name="line.530"></a>
-<span class="sourceLineNo">531</span> restoreLocks();<a name="line.531"></a>
-<span class="sourceLineNo">532</span><a name="line.532"></a>
-<span class="sourceLineNo">533</span> // 5. Push the procedures to the timeout executor<a name="line.533"></a>
-<span class="sourceLineNo">534</span> waitingTimeoutList.forEach(proc -> {<a name="line.534"></a>
-<span class="sourceLineNo">535</span> proc.afterReplay(getEnvironment());<a name="line.535"></a>
-<span class="sourceLineNo">536</span> timeoutExecutor.add(proc);<a name="line.536"></a>
-<span class="sourceLineNo">537</span> });<a name="line.537"></a>
-<span class="sourceLineNo">538</span><a name="line.538"></a>
-<span class="sourceLineNo">539</span> // 6. Push the procedure to the scheduler<a name="line.539"></a>
-<span class="sourceLineNo">540</span> failedList.forEach(scheduler::addBack);<a name="line.540"></a>
-<span class="sourceLineNo">541</span> runnableList.forEach(p -> {<a name="line.541"></a>
-<span class="sourceLineNo">542</span> p.afterReplay(getEnvironment());<a name="line.542"></a>
-<span class="sourceLineNo">543</span> if (!p.hasParent()) {<a name="line.543"></a>
-<span class="sourceLineNo">544</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.544"></a>
-<span class="sourceLineNo">545</span> }<a name="line.545"></a>
-<span class="sourceLineNo">546</span> scheduler.addBack(p);<a name="line.546"></a>
-<span class="sourceLineNo">547</span> });<a name="line.547"></a>
-<span class="sourceLineNo">548</span> // After all procedures put into the queue, signal the worker threads.<a name="line.548"></a>
-<span class="sourceLineNo">549</span> // Otherwise, there is a race condition. See HBASE-21364.<a name="line.549"></a>
-<span class="sourceLineNo">550</span> scheduler.signalAll();<a name="line.550"></a>
-<span class="sourceLineNo">551</span> }<a name="line.551"></a>
-<span class="sourceLineNo">552</span><a name="line.552"></a>
-<span class="sourceLineNo">553</span> /**<a name="line.553"></a>
-<span class="sourceLineNo">554</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.554"></a>
-<span class="sourceLineNo">555</span> * <p/><a name="line.555"></a>
-<span class="sourceLineNo">556</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.556"></a>
-<span class="sourceLineNo">557</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.557"></a>
-<span class="sourceLineNo">558</span> * pending and in-progress procedures.<a name="line.558"></a>
-<span class="sourceLineNo">559</span> * @param numThreads number of threads available for procedure execution.<a name="line.559"></a>
-<span class="sourceLineNo">560</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.560"></a>
-<span class="sourceLineNo">561</span> * is found on replay. otherwise false.<a name="line.561"></a>
-<span class="sourceLineNo">562</span> */<a name="line.562"></a>
-<span class="sourceLineNo">563</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.563"></a>
-<span class="sourceLineNo">564</span> // We have numThreads executor + one timer thread used for timing out<a name="line.564"></a>
-<span class="sourceLineNo">565</span> // procedures and triggering periodic procedures.<a name="line.565"></a>
-<span class="sourceLineNo">566</span> this.corePoolSize = numThreads;<a name="line.566"></a>
-<span class="sourceLineNo">567</span> this.maxPoolSize = 10 * numThreads;<a name="line.567"></a>
-<span class="sourceLineNo">568</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.568"></a>
-<span class="sourceLineNo">569</span> corePoolSize, maxPoolSize);<a name="line.569"></a>
-<span class="sourceLineNo">570</span><a name="line.570"></a>
-<span class="sourceLineNo">571</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.571"></a>
-<span class="sourceLineNo">572</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, "ProcExecTimeout");<a name="line.572"></a>
-<span class="sourceLineNo">573</span> this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, threadGroup, "WorkerMonitor");<a name="line.573"></a>
-<span class="sourceLineNo">574</span><a name="line.574"></a>
-<span class="sourceLineNo">575</span> // Create the workers<a name="line.575"></a>
-<span class="sourceLineNo">576</span> workerId.set(0);<a name="line.576"></a>
-<span class="sourceLineNo">577</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.577"></a>
-<span class="sourceLineNo">578</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.578"></a>
-<span class="sourceLineNo">579</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.579"></a>
-<span class="sourceLineNo">580</span> }<a name="line.580"></a>
-<span class="sourceLineNo">581</span><a name="line.581"></a>
-<span class="sourceLineNo">582</span> long st, et;<a name="line.582"></a>
-<span class="sourceLineNo">583</span><a name="line.583"></a>
-<span class="sourceLineNo">584</span> // Acquire the store lease.<a name="line.584"></a>
-<span class="sourceLineNo">585</span> st = System.nanoTime();<a name="line.585"></a>
-<span class="sourceLineNo">586</span> store.recoverLease();<a name="line.586"></a>
-<span class="sourceLineNo">587</span> et = System.nanoTime();<a name="line.587"></a>
-<span class="sourceLineNo">588</span> LOG.info("Recovered {} lease in {}", store.getClass().getSimpleName(),<a name="line.588"></a>
-<span class="sourceLineNo">589</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.589"></a>
-<span class="sourceLineNo">590</span><a name="line.590"></a>
-<span class="sourceLineNo">591</span> // start the procedure scheduler<a name="line.591"></a>
-<span class="sourceLineNo">592</span> scheduler.start();<a name="line.592"></a>
-<span class="sourceLineNo">593</span><a name="line.593"></a>
-<span class="sourceLineNo">594</span> // TODO: Split in two steps.<a name="line.594"></a>
-<span class="sourceLineNo">595</span> // TODO: Handle corrupted procedures (currently just a warn)<a name="line.595"></a>
-<span class="sourceLineNo">596</span> // The first one will make sure that we have the latest id,<a name="line.596"></a>
-<span class="sourceLineNo">597</span> // so we can start the threads and accept new procedures.<a name="line.597"></a>
-<span class="sourceLineNo">598</span> // The second step will do the actual load of old procedures.<a name="line.598"></a>
-<span class="sourceLineNo">599</span> st = System.nanoTime();<a name="line.599"></a>
-<span class="sourceLineNo">600</span> load(abortOnCorruption);<a name="line.600"></a>
-<span class="sourceLineNo">601</span> et = System.nanoTime();<a name="line.601"></a>
-<span class="sourceLineNo">602</span> LOG.info("Loaded {} in {}", store.getClass().getSimpleName(),<a name="line.602"></a>
-<span class="sourceLineNo">603</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.603"></a>
-<span class="sourceLineNo">604</span> }<a name="line.604"></a>
-<span class="sourceLineNo">605</span><a name="line.605"></a>
-<span class="sourceLineNo">606</span> /**<a name="line.606"></a>
-<span class="sourceLineNo">607</span> * Start the workers.<a name="line.607"></a>
-<span class="sourceLineNo">608</span> */<a name="line.608"></a>
-<span class="sourceLineNo">609</span> public void startWorkers() throws IOException {<a name="line.609"></a>
-<span class="sourceLineNo">610</span> if (!running.compareAndSet(false, true)) {<a name="line.610"></a>
-<span class="sourceLineNo">611</span> LOG.warn("Already running");<a name="line.611"></a>
-<span class="sourceLineNo">612</span> return;<a name="line.612"></a>
-<span class="sourceLineNo">613</span> }<a name="line.613"></a>
-<span class="sourceLineNo">614</span> // Start the executors. Here we must have the lastProcId set.<a name="line.614"></a>
-<span class="sourceLineNo">615</span> LOG.trace("Start workers {}", workerThreads.size());<a name="line.615"></a>
-<span class="sourceLineNo">616</span> timeoutExecutor.start();<a name="line.616"></a>
-<span class="sourceLineNo">617</span> workerMonitorExecutor.start();<a name="line.617"></a>
-<span class="sourceLineNo">618</span> for (WorkerThread worker: workerThreads) {<a name="line.618"></a>
-<span class="sourceLineNo">619</span> worker.start();<a name="line.619"></a>
-<span class="sourceLineNo">620</span> }<a name="line.620"></a>
-<span class="sourceLineNo">621</span><a name="line.621"></a>
-<span class="sourceLineNo">622</span> // Internal chores<a name="line.622"></a>
-<span class="sourceLineNo">623</span> workerMonitorExecutor.add(new WorkerMonitor());<a name="line.623"></a>
-<span class="sourceLineNo">624</span><a name="line.624"></a>
-<span class="sourceLineNo">625</span> // Add completed cleaner chore<a name="line.625"></a>
-<span class="sourceLineNo">626</span> addChore(new CompletedProcedureCleaner<>(conf, store, procExecutionLock, completed,<a name="line.626"></a>
-<span class="sourceLineNo">627</span> nonceKeysToProcIdsMap));<a name="line.627"></a>
-<span class="sourceLineNo">628</span> }<a name="line.628"></a>
-<span class="sourceLineNo">629</span><a name="line.629"></a>
-<span class="sourceLineNo">630</span> public void stop() {<a name="line.630"></a>
-<span class="sourceLineNo">631</span> if (!running.getAndSet(false)) {<a name="line.631"></a>
-<span class="sourceLineNo">632</span> return;<a name="line.632"></a>
-<span class="sourceLineNo">633</span> }<a name="line.633"></a>
-<span class="sourceLineNo">634</span><a name="line.634"></a>
-<span class="sourceLineNo">635</span> LOG.info("Stopping");<a name="line.635"></a>
-<span class="sourceLineNo">636</span> scheduler.stop();<a name="line.636"></a>
-<span class="sourceLineNo">637</span> timeoutExecutor.sendStopSignal();<a name="line.637"></a>
-<span class="sourceLineNo">638</span> workerMonitorExecutor.sendStopSignal();<a name="line.638"></a>
-<span class="sourceLineNo">639</span> }<a name="line.639"></a>
-<span class="sourceLineNo">640</span><a name="line.640"></a>
-<span class="sourceLineNo">641</span> @VisibleForTesting<a name="line.641"></a>
-<span class="sourceLineNo">642</span> public void join() {<a name="line.642"></a>
-<span class="sourceLineNo">643</span> assert !isRunning() : "expected not running";<a name="line.643"></a>
-<span class="sourceLineNo">644</span><a name="line.644"></a>
-<span class="sourceLineNo">645</span> // stop the timeout executor<a name="line.645"></a>
-<span class="sourceLineNo">646</span> timeoutExecutor.awaitTermination();<a name="line.646"></a>
-<span class="sourceLineNo">647</span> // stop the work monitor executor<a name="line.647"></a>
-<span class="sourceLineNo">648</span> workerMonitorExecutor.awaitTermination();<a name="line.648"></a>
-<span class="sourceLineNo">649</span><a name="line.649"></a>
-<span class="sourceLineNo">650</span> // stop the worker threads<a name="line.650"></a>
-<span class="sourceLineNo">651</span> for (WorkerThread worker: workerThreads) {<a name="line.651"></a>
-<span class="sourceLineNo">652</span> worker.awaitTermination();<a name="line.652"></a>
-<span class="sourceLineNo">653</span> }<a name="line.653"></a>
-<span class="sourceLineNo">654</span><a name="line.654"></a>
-<span class="sourceLineNo">655</span> // Destroy the Thread Group for the executors<a name="line.655"></a>
-<span class="sourceLineNo">656</span> // TODO: Fix. #join is not place to destroy resources.<a name="line.656"></a>
-<span class="sourceLineNo">657</span> try {<a name="line.657"></a>
-<span class="sourceLineNo">658</span> threadGroup.destroy();<a name="line.658"></a>
-<span class="sourceLineNo">659</span> } catch (IllegalThreadStateException e) {<a name="line.659"></a>
-<span class="sourceLineNo">660</span> LOG.error("ThreadGroup {} contains running threads; {}: See STDOUT",<a name="line.660"></a>
-<span class="sourceLineNo">661</span> this.threadGroup, e.getMessage());<a name="line.661"></a>
-<span class="sourceLineNo">662</span> // This dumps list of threads on STDOUT.<a name="line.662"></a>
-<span class="sourceLineNo">663</span> this.threadGroup.list();<a name="line.663"></a>
-<span class="sourceLineNo">664</span> }<a name="line.664"></a>
-<span class="sourceLineNo">665</span><a name="line.665"></a>
-<span class="sourceLineNo">666</span> // reset the in-memory state for testing<a name="line.666"></a>
-<span class="sourceLineNo">667</span> completed.clear();<a name="line.667"></a>
-<span class="sourceLineNo">668</span> rollbackStack.clear();<a name="line.668"></a>
-<span class="sourceLineNo">669</span> procedures.clear();<a name="line.669"></a>
-<span class="sourceLineNo">670</span> nonceKeysToProcIdsMap.clear();<a name="line.670"></a>
-<span class="sourceLineNo">671</span> scheduler.clear();<a name="line.671"></a>
-<span class="sourceLineNo">672</span> lastProcId.set(-1);<a name="line.672"></a>
-<span class="sourceLineNo">673</span> }<a name="line.673"></a>
-<span class="sourceLineNo">674</span><a name="line.674"></a>
-<span class="sourceLineNo">675</span> public void refreshConfiguration(final Configuration conf) {<a name="line.675"></a>
-<span class="sourceLineNo">676</span> this.conf = conf;<a name="line.676"></a>
-<span class="sourceLineNo">677</span> setKeepAliveTime(conf.getLong(WORKER_KEEP_ALIVE_TIME_CONF_KEY,<a name="line.677"></a>
-<span class="sourceLineNo">678</span> DEFAULT_WORKER_KEEP_ALIVE_TIME), TimeUnit.MILLISECONDS);<a name="line.678"></a>
-<span class="sourceLineNo">679</span> }<a name="line.679"></a>
-<span class="sourceLineNo">680</span><a name="line.680"></a>
-<span class="sourceLineNo">681</span> // ==========================================================================<a name="line.681"></a>
-<span class="sourceLineNo">682</span> // Accessors<a name="line.682"></a>
-<span class="sourceLineNo">683</span> // ==========================================================================<a name="line.683"></a>
-<span class="sourceLineNo">684</span> public boolean isRunning() {<a name="line.684"></a>
-<span class="sourceLineNo">685</span> return running.get();<a name="line.685"></a>
-<span class="sourceLineNo">686</span> }<a name="line.686"></a>
-<span class="sourceLineNo">687</span><a name="line.687"></a>
-<span class="sourceLineNo">688</span> /**<a name="line.688"></a>
-<span class="sourceLineNo">689</span> * @return the current number of worker threads.<a name="line.689"></a>
-<span class="sourceLineNo">690</span> */<a name="line.690"></a>
-<span class="sourceLineNo">691</span> public int getWorkerThreadCount() {<a name="line.691"></a>
-<span class="sourceLineNo">692</span> return workerThreads.size();<a name="line.692"></a>
-<span class="sourceLineNo">693</span> }<a name="line.693"></a>
-<span class="sourceLineNo">694</span><a name="line.694"></a>
-<span class="sourceLineNo">695</span> /**<a name="line.695"></a>
-<span class="sourceLineNo">696</span> * @return the core pool size settings.<a name="line.696"></a>
-<span class="sourceLineNo">697</span> */<a name="line.697"></a>
-<span class="sourceLineNo">698</span> public int getCorePoolSize() {<a name="line.698"></a>
-<span class="sourceLineNo">699</span> return corePoolSize;<a name="line.699"></a>
-<span class="sourceLineNo">700</span> }<a name="line.700"></a>
-<span class="sourceLineNo">701</span><a name="line.701"></a>
-<span class="sourceLineNo">702</span> public int getActiveExecutorCount() {<a name="line.702"></a>
-<span class="sourceLineNo">703</span> return activeExecutorCount.get();<a name="line.703"></a>
-<span class="sourceLineNo">704</span> }<a name="line.704"></a>
-<span class="sourceLineNo">705</span><a name="line.705"></a>
-<span class="sourceLineNo">706</span> public TEnvironment getEnvironment() {<a name="line.706"></a>
-<span class="sourceLineNo">707</span> return this.environment;<a name="line.707"></a>
-<span class="sourceLineNo">708</span> }<a name="line.708"></a>
-<span class="sourceLineNo">709</span><a name="line.709"></a>
-<span class="sourceLineNo">710</span> public ProcedureStore getStore() {<a name="line.710"></a>
-<span class="sourceLineNo">711</span> return this.store;<a name="line.711"></a>
-<span class="sourceLineNo">712</span> }<a name="line.712"></a>
-<span class="sourceLineNo">713</span><a name="line.713"></a>
-<span class="sourceLineNo">714</span> ProcedureScheduler getScheduler() {<a name="line.714"></a>
-<span class="sourceLineNo">715</span> return scheduler;<a name="line.715"></a>
-<span class="sourceLineNo">716</span> }<a name="line.716"></a>
-<span class="sourceLineNo">717</span><a name="line.717"></a>
-<span class="sourceLineNo">718</span> public void setKeepAliveTime(final long keepAliveTime, final TimeUnit timeUnit) {<a name="line.718"></a>
-<span class="sourceLineNo">719</span> this.keepAliveTime = timeUnit.toMillis(keepAliveTime);<a name="line.719"></a>
-<span class="sourceLineNo">720</span> this.scheduler.signalAll();<a name="line.720"></a>
-<span class="sourceLineNo">721</span> }<a name="line.721"></a>
-<span class="sourceLineNo">722</span><a name="line.722"></a>
-<span class="sourceLineNo">723</span> public long getKeepAliveTime(final TimeUnit timeUnit) {<a name="line.723"></a>
-<span class="sourceLineNo">724</span> return timeUnit.convert(keepAliveTime, TimeUnit.MILLISECONDS);<a name="line.724"></a>
-<span class="sourceLineNo">725</span> }<a name="line.725"></a>
-<span class="sourceLineNo">726</span><a name="line.726"></a>
-<span class="sourceLineNo">727</span> // ==========================================================================<a name="line.727"></a>
-<span class="sourceLineNo">728</span> // Submit/Remove Chores<a name="line.728"></a>
-<span class="sourceLineNo">729</span> // ==========================================================================<a name="line.729"></a>
-<span class="sourceLineNo">730</span><a name="line.730"></a>
-<span class="sourceLineNo">731</span> /**<a name="line.731"></a>
-<span class="sourceLineNo">732</span> * Add a chore procedure to the executor<a name="line.732"></a>
-<span class="sourceLineNo">733</span> * @param chore the chore to add<a name="line.733"></a>
-<span class="sourceLineNo">734</span> */<a name="line.734"></a>
-<span class="sourceLineNo">735</span> public void addChore(ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.735"></a>
-<span class="sourceLineNo">736</span> chore.setState(ProcedureState.WAITING_TIMEOUT);<a name="line.736"></a>
-<span class="sourceLineNo">737</span> timeoutExecutor.add(chore);<a name="line.737"></a>
-<span class="sourceLineNo">738</span> }<a name="line.738"></a>
-<span class="sourceLineNo">739</span><a name="line.739"></a>
-<span class="sourceLineNo">740</span> /**<a name="line.740"></a>
-<span class="sourceLineNo">741</span> * Remove a chore procedure from the executor<a name="line.741"></a>
-<span class="sourceLineNo">742</span> * @param chore the chore to remove<a name="line.742"></a>
-<span class="sourceLineNo">743</span> * @return whether the chore is removed, or it will be removed later<a name="line.743"></a>
-<span class="sourceLineNo">744</span> */<a name="line.744"></a>
-<span class="sourceLineNo">745</span> public boolean removeChore(ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.745"></a>
-<span class="sourceLineNo">746</span> chore.setState(ProcedureState.SUCCESS);<a name="line.746"></a>
-<span class="sourceLineNo">747</span> return timeoutExecutor.remove(chore);<a name="line.747"></a>
-<span class="sourceLineNo">748</span> }<a name="line.748"></a>
-<span class="sourceLineNo">749</span><a name="line.749"></a>
-<span class="sourceLineNo">750</span> // ==========================================================================<a name="line.750"></a>
-<span class="sourceLineNo">751</span> // Nonce Procedure helpers<a name="line.751"></a>
-<span class="sourceLineNo">752</span> // ==========================================================================<a name="line.752"></a>
-<span class="sourceLineNo">753</span> /**<a name="line.753"></a>
-<span class="sourceLineNo">754</span> * Create a NonceKey from the specified nonceGroup and nonce.<a name="line.754"></a>
-<span class="sourceLineNo">755</span> * @param nonceGroup the group to use for the {@link NonceKey}<a name="line.755"></a>
-<span class="sourceLineNo">756</span> * @param nonce the nonce to use in the {@link NonceKey}<a name="line.756"></a>
-<span class="sourceLineNo">757</span> * @return the generated NonceKey<a name="line.757"></a>
-<span class="sourceLineNo">758</span> */<a name="line.758"></a>
-<span class="sourceLineNo">759</span> public NonceKey createNonceKey(final long nonceGroup, final long nonce) {<a name="line.759"></a>
-<span class="sourceLineNo">760</span> return (nonce == HConstants.NO_NONCE) ? null : new NonceKey(nonceGroup, nonce);<a name="line.760"></a>
-<span class="sourceLineNo">761</span> }<a name="line.761"></a>
-<span class="sourceLineNo">762</span><a name="line.762"></a>
-<span class="sourceLineNo">763</span> /**<a name="line.763"></a>
-<span class="sourceLineNo">764</span> * Register a nonce for a procedure that is going to be submitted.<a name="line.764"></a>
-<span class="sourceLineNo">765</span> * A procId will be reserved and on submitProcedure(),<a name="line.765"></a>
-<span class="sourceLineNo">766</span> * the procedure with the specified nonce will take the reserved ProcId.<a name="line.766"></a>
-<span class="sourceLineNo">767</span> * If someone already reserved the nonce, this method will return the procId reserved,<a name="line.767"></a>
-<span class="sourceLineNo">768</span> * otherwise an invalid procId will be returned. and the caller should procede<a name="line.768"></a>
-<span class="sourceLineNo">769</span> * and submit the procedure.<a name="line.769"></a>
-<span class="sourceLineNo">770</span> *<a name="line.770"></a>
-<span class="sourceLineNo">771</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.771"></a>
-<span class="sourceLineNo">772</span> * @return the procId associated with the nonce, if any otherwise an invalid procId.<a name="line.772"></a>
-<span class="sourceLineNo">773</span> */<a name="line.773"></a>
-<span class="sourceLineNo">774</span> public long registerNonce(final NonceKey nonceKey) {<a name="line.774"></a>
-<span class="sourceLineNo">775</span> if (nonceKey == null) {<a name="line.775"></a>
-<span class="sourceLineNo">776</span> return -1;<a name="line.776"></a>
-<span class="sourceLineNo">777</span> }<a name="line.777"></a>
-<span class="sourceLineNo">778</span><a name="line.778"></a>
-<span class="sourceLineNo">779</span> // check if we have already a Reserved ID for the nonce<a name="line.779"></a>
-<span class="sourceLineNo">780</span> Long oldProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.780"></a>
-<span class="sourceLineNo">781</span> if (oldProcId == null) {<a name="line.781"></a>
-<span class="sourceLineNo">782</span> // reserve a new Procedure ID, this will be associated with the nonce<a name="line.782"></a>
-<span class="sourceLineNo">783</span> // and the procedure submitted with the specified nonce will use this ID.<a name="line.783"></a>
-<span class="sourceLineNo">784</span> final long newProcId = nextProcId();<a name="line.784"></a>
-<span class="sourceLineNo">785</span> oldProcId = nonceKeysToProcIdsMap.putIfAbsent(nonceKey, newProcId);<a name="line.785"></a>
-<span class="sourceLineNo">786</span> if (oldProcId == null) {<a name="line.786"></a>
-<span class="sourceLineNo">787</span> return -1;<a name="line.787"></a>
-<span class="sourceLineNo">788</span> }<a name="line.788"></a>
-<span class="sourceLineNo">789</span> }<a name="line.789"></a>
-<span class="sourceLineNo">790</span><a name="line.790"></a>
-<span class="sourceLineNo">791</span> // we found a registered nonce, but the procedure may not have been submitted yet.<a name="line.791"></a>
-<span class="sourceLineNo">792</span> // since the client expect the procedure to be submitted, spin here until it is.<a name="line.792"></a>
-<span class="sourceLineNo">793</span> final boolean traceEnabled = LOG.isTraceEnabled();<a name="line.793"></a>
-<span class="sourceLineNo">794</span> while (isRunning() &&<a name="line.794"></a>
-<span class="sourceLineNo">795</span> !(procedures.containsKey(oldProcId) || completed.containsKey(oldProcId)) &&<a name="line.795"></a>
-<span class="sourceLineNo">796</span> nonceKeysToProcIdsMap.containsKey(nonceKey)) {<a name="line.796"></a>
-<span class="sourceLineNo">797</span> if (traceEnabled) {<a name="line.797"></a>
-<span class="sourceLineNo">798</span> LOG.trace("Waiting for pid=" + oldProcId.longValue() + " to be submitted");<a name="line.798"></a>
-<span class="sourceLineNo">799</span> }<a name="line.799"></a>
-<span class="sourceLineNo">800</span> Threads.sleep(100);<a name="line.800"></a>
-<span class="sourceLineNo">801</span> }<a name="line.801"></a>
-<span class="sourceLineNo">802</span> return oldProcId.longValue();<a name="line.802"></a>
-<span class="sourceLineNo">803</span> }<a name="line.803"></a>
-<span class="sourceLineNo">804</span><a name="line.804"></a>
-<span class="sourceLineNo">805</span> /**<a name="line.805"></a>
-<span class="sourceLineNo">806</span> * Remove the NonceKey if the procedure was not submitted to the executor.<a name="line.806"></a>
-<span class="sourceLineNo">807</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.807"></a>
-<span class="sourceLineNo">808</span> */<a name="line.808"></a>
-<span class="sourceLineNo">809</span> public void unregisterNonceIfProcedureWasNotSubmitted(final NonceKey nonceKey) {<a name="line.809"></a>
-<span class="sourceLineNo">810</span> if (nonceKey == null) {<a name="line.810"></a>
-<span class="sourceLineNo">811</span> return;<a name="line.811"></a>
-<span class="sourceLineNo">812</span> }<a name="line.812"></a>
-<span class="sourceLineNo">813</span><a name="line.813"></a>
-<span class="sourceLineNo">814</span> final Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.814"></a>
-<span class="sourceLineNo">815</span> if (procId == null) {<a name="line.815"></a>
-<span class="sourceLineNo">816</span> return;<a name="line.816"></a>
-<span class="sourceLineNo">817</span> }<a name="line.817"></a>
-<span class="sourceLineNo">818</span><a name="line.818"></a>
-<span class="sourceLineNo">819</span> // if the procedure was not submitted, remove the nonce<a name="line.819"></a>
-<span class="sourceLineNo">820</span> if (!(procedures.containsKey(procId) || completed.containsKey(procId))) {<a name="line.820"></a>
-<span class="sourceLineNo">821</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.821"></a>
-<span class="sourceLineNo">822</span> }<a name="line.822"></a>
-<span class="sourceLineNo">823</span> }<a name="line.823"></a>
-<span class="sourceLineNo">824</span><a name="line.824"></a>
-<span class="sourceLineNo">825</span> /**<a name="line.825"></a>
-<span class="sourceLineNo">826</span> * If the failure failed before submitting it, we may want to give back the<a name="line.826"></a>
-<span class="sourceLineNo">827</span> * same error to the requests with the same nonceKey.<a name="line.827"></a>
-<span class="sourceLineNo">828</span> *<a name="line.828"></a>
-<span class="sourceLineNo">829</span> * @param nonceKey A unique identifier for this operation from the client or process<a name="line.829"></a>
-<span class="sourceLineNo">830</span> * @param procName name of the procedure, used to inform the user<a name="line.830"></a>
-<span class="sourceLineNo">831</span> * @param procOwner name of the owner of the procedure, used to inform the user<a name="line.831"></a>
-<span class="sourceLineNo">832</span> * @param exception the failure to report to the user<a name="line.832"></a>
-<span class="sourceLineNo">833</span> */<a name="line.833"></a>
-<span class="sourceLineNo">834</span> public void setFailureResultForNonce(NonceKey nonceKey, String procName, User procOwner,<a name="line.834"></a>
-<span class="sourceLineNo">835</span> IOException exception) {<a name="line.835"></a>
-<span class="sourceLineNo">836</span> if (nonceKey == null) {<a name="line.836"></a>
-<span class="sourceLineNo">837</span> return;<a name="line.837"></a>
-<span class="sourceLineNo">838</span> }<a name="line.838"></a>
-<span class="sourceLineNo">839</span><a name="line.839"></a>
-<span class="sourceLineNo">840</span> Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.840"></a>
-<span class="sourceLineNo">841</span> if (procId == null || completed.containsKey(procId)) {<a name="line.841"></a>
-<span class="sourceLineNo">842</span> return;<a name="line.842"></a>
-<span class="sourceLineNo">843</span> }<a name="line.843"></a>
-<span class="sourceLineNo">844</span><a name="line.844"></a>
-<span class="sourceLineNo">845</span> completed.computeIfAbsent(procId, (key) -> {<a name="line.845"></a>
-<span class="sourceLineNo">846</span> Procedure<TEnvironment> proc = new FailedProcedure<>(procId.longValue(),<a name="line.846"></a>
-<span class="sourceLineNo">847</span> procName, procOwner, nonceKey, exception);<a name="line.847"></a>
-<span class="sourceLineNo">848</span><a name="line.848"></a>
-<span class="sourceLineNo">849</span> return new CompletedProcedureRetainer<>(proc);<a name="line.849"></a>
-<span class="sourceLineNo">850</span> });<a name="line.850"></a>
-<span class="sourceLineNo">851</span> }<a name="line.851"></a>
-<span class="sourceLineNo">852</span><a name="line.852"></a>
-<span class="sourceLineNo">853</span> // ==========================================================================<a name="line.853"></a>
-<span class="sourceLineNo">854</span> // Submit/Abort Procedure<a name="line.854"></a>
-<span class="sourceLineNo">855</span> // ==========================================================================<a name="line.855"></a>
-<span class="sourceLineNo">856</span> /**<a name="line.856"></a>
-<span class="sourceLineNo">857</span> * Add a new root-procedure to the executor.<a name="line.857"></a>
-<span class="sourceLineNo">858</span> * @param proc the new procedure to execute.<a name="line.858"></a>
-<span class="sourceLineNo">859</span> * @return the procedure id, that can be used to monitor the operation<a name="line.859"></a>
-<span class="sourceLineNo">860</span> */<a name="line.860"></a>
-<span class="sourceLineNo">861</span> public long submitProcedure(Procedure<TEnvironment> proc) {<a name="line.861"></a>
-<span class="sourceLineNo">862</span> return submitProcedure(proc, null);<a name="line.862"></a>
-<span class="sourceLineNo">863</span> }<a name="line.863"></a>
-<span class="sourceLineNo">864</span><a name="line.864"></a>
-<span class="sourceLineNo">865</span> /**<a name="line.865"></a>
-<span class="sourceLineNo">866</span> * Bypass a procedure. If the procedure is set to bypass, all the logic in<a name="line.866"></a>
-<span class="sourceLineNo">867</span> * execute/rollback will be ignored and it will return success, whatever.<a name="line.867"></a>
-<span class="sourceLineNo">868</span> * It is used to recover buggy stuck procedures, releasing the lock resources<a name="line.868"></a>
-<span class="sourceLineNo">869</span> * and letting other procedures run. Bypassing one procedure (and its ancestors will<a name="line.869"></a>
-<span class="sourceLineNo">870</span> * be bypassed automatically) may leave the cluster in a middle state, e.g. region<a name="line.870"></a>
-<span class="sourceLineNo">871</span> * not assigned, or some hdfs files left behind. After getting rid of those stuck procedures,<a name="line.871"></a>
-<span class="sourceLineNo">872</span> * the operators may have to do some clean up on hdfs or schedule some assign procedures<a name="line.872"></a>
-<span class="sourceLineNo">873</span> * to let region online. DO AT YOUR OWN RISK.<a name="line.873"></a>
-<span class="sourceLineNo">874</span> * <p><a name="line.874"></a>
-<span class="sourceLineNo">875</span> * A procedure can be bypassed only if<a name="line.875"></a>
-<span class="sourceLineNo">876</span> * 1. The procedure is in state of RUNNABLE, WAITING, WAITING_TIMEOUT<a name="line.876"></a>
-<span class="sourceLineNo">877</span> * or it is a root procedure without any child.<a name="line.877"></a>
-<span class="sourceLineNo">878</span> * 2. No other worker thread is executing it<a name="line.878"></a>
-<span class="sourceLineNo">879</span> * 3. No child procedure has been submitted<a name="line.879"></a>
-<span class="sourceLineNo">880</span> *<a name="line.880"></a>
+<span class="sourceLineNo">020</span>import edu.umd.cs.findbugs.annotations.Nullable;<a name="line.20"></a>
+<span class="sourceLineNo">021</span>import java.io.IOException;<a name="line.21"></a>
+<span class="sourceLineNo">022</span>import java.io.UncheckedIOException;<a name="line.22"></a>
+<span class="sourceLineNo">023</span>import java.util.ArrayDeque;<a name="line.23"></a>
+<span class="sourceLineNo">024</span>import java.util.ArrayList;<a name="line.24"></a>
+<span class="sourceLineNo">025</span>import java.util.Arrays;<a name="line.25"></a>
+<span class="sourceLineNo">026</span>import java.util.Collection;<a name="line.26"></a>
+<span class="sourceLineNo">027</span>import java.util.Deque;<a name="line.27"></a>
+<span class="sourceLineNo">028</span>import java.util.HashSet;<a name="line.28"></a>
+<span class="sourceLineNo">029</span>import java.util.List;<a name="line.29"></a>
+<span class="sourceLineNo">030</span>import java.util.Set;<a name="line.30"></a>
+<span class="sourceLineNo">031</span>import java.util.concurrent.ConcurrentHashMap;<a name="line.31"></a>
+<span class="sourceLineNo">032</span>import java.util.concurrent.CopyOnWriteArrayList;<a name="line.32"></a>
+<span class="sourceLineNo">033</span>import java.util.concurrent.Executor;<a name="line.33"></a>
+<span class="sourceLineNo">034</span>import java.util.concurrent.Executors;<a name="line.34"></a>
+<span class="sourceLineNo">035</span>import java.util.concurrent.TimeUnit;<a name="line.35"></a>
+<span class="sourceLineNo">036</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.36"></a>
+<span class="sourceLineNo">037</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.37"></a>
+<span class="sourceLineNo">038</span>import java.util.concurrent.atomic.AtomicLong;<a name="line.38"></a>
+<span class="sourceLineNo">039</span>import java.util.stream.Collectors;<a name="line.39"></a>
+<span class="sourceLineNo">040</span>import java.util.stream.Stream;<a name="line.40"></a>
+<span class="sourceLineNo">041</span>import org.apache.hadoop.conf.Configuration;<a name="line.41"></a>
+<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.HConstants;<a name="line.42"></a>
+<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.exceptions.IllegalArgumentIOException;<a name="line.43"></a>
+<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.log.HBaseMarkers;<a name="line.44"></a>
+<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.procedure2.Procedure.LockState;<a name="line.45"></a>
+<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore;<a name="line.46"></a>
+<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;<a name="line.47"></a>
+<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureStoreListener;<a name="line.48"></a>
+<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.procedure2.util.StringUtils;<a name="line.49"></a>
+<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.security.User;<a name="line.50"></a>
+<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.51"></a>
+<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.util.IdLock;<a name="line.52"></a>
+<span class="sourceLineNo">053</span>import org.apache.hadoop.hbase.util.NonceKey;<a name="line.53"></a>
+<span class="sourceLineNo">054</span>import org.apache.hadoop.hbase.util.Threads;<a name="line.54"></a>
+<span class="sourceLineNo">055</span>import org.apache.yetus.audience.InterfaceAudience;<a name="line.55"></a>
+<span class="sourceLineNo">056</span>import org.slf4j.Logger;<a name="line.56"></a>
+<span class="sourceLineNo">057</span>import org.slf4j.LoggerFactory;<a name="line.57"></a>
+<span class="sourceLineNo">058</span><a name="line.58"></a>
+<span class="sourceLineNo">059</span>import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;<a name="line.59"></a>
+<span class="sourceLineNo">060</span>import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;<a name="line.60"></a>
+<span class="sourceLineNo">061</span>import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;<a name="line.61"></a>
+<span class="sourceLineNo">062</span><a name="line.62"></a>
+<span class="sourceLineNo">063</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos.ProcedureState;<a name="line.63"></a>
+<span class="sourceLineNo">064</span><a name="line.64"></a>
+<span class="sourceLineNo">065</span>/**<a name="line.65"></a>
+<span class="sourceLineNo">066</span> * Thread Pool that executes the submitted procedures.<a name="line.66"></a>
+<span class="sourceLineNo">067</span> * The executor has a ProcedureStore associated.<a name="line.67"></a>
+<span class="sourceLineNo">068</span> * Each operation is logged and on restart the pending procedures are resumed.<a name="line.68"></a>
+<span class="sourceLineNo">069</span> *<a name="line.69"></a>
+<span class="sourceLineNo">070</span> * Unless the Procedure code throws an error (e.g. invalid user input)<a name="line.70"></a>
+<span class="sourceLineNo">071</span> * the procedure will complete (at some point in time), On restart the pending<a name="line.71"></a>
+<span class="sourceLineNo">072</span> * procedures are resumed and the once failed will be rolledback.<a name="line.72"></a>
+<span class="sourceLineNo">073</span> *<a name="line.73"></a>
+<span class="sourceLineNo">074</span> * The user can add procedures to the executor via submitProcedure(proc)<a name="line.74"></a>
+<span class="sourceLineNo">075</span> * check for the finished state via isFinished(procId)<a name="line.75"></a>
+<span class="sourceLineNo">076</span> * and get the result via getResult(procId)<a name="line.76"></a>
+<span class="sourceLineNo">077</span> */<a name="line.77"></a>
+<span class="sourceLineNo">078</span>@InterfaceAudience.Private<a name="line.78"></a>
+<span class="sourceLineNo">079</span>public class ProcedureExecutor<TEnvironment> {<a name="line.79"></a>
+<span class="sourceLineNo">080</span> private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);<a name="line.80"></a>
+<span class="sourceLineNo">081</span><a name="line.81"></a>
+<span class="sourceLineNo">082</span> public static final String CHECK_OWNER_SET_CONF_KEY = "hbase.procedure.check.owner.set";<a name="line.82"></a>
+<span class="sourceLineNo">083</span> private static final boolean DEFAULT_CHECK_OWNER_SET = false;<a name="line.83"></a>
+<span class="sourceLineNo">084</span><a name="line.84"></a>
+<span class="sourceLineNo">085</span> public static final String WORKER_KEEP_ALIVE_TIME_CONF_KEY =<a name="line.85"></a>
+<span class="sourceLineNo">086</span> "hbase.procedure.worker.keep.alive.time.msec";<a name="line.86"></a>
+<span class="sourceLineNo">087</span> private static final long DEFAULT_WORKER_KEEP_ALIVE_TIME = TimeUnit.MINUTES.toMillis(1);<a name="line.87"></a>
+<span class="sourceLineNo">088</span><a name="line.88"></a>
+<span class="sourceLineNo">089</span> public static final String EVICT_TTL_CONF_KEY = "hbase.procedure.cleaner.evict.ttl";<a name="line.89"></a>
+<span class="sourceLineNo">090</span> static final int DEFAULT_EVICT_TTL = 15 * 60000; // 15min<a name="line.90"></a>
+<span class="sourceLineNo">091</span><a name="line.91"></a>
+<span class="sourceLineNo">092</span> public static final String EVICT_ACKED_TTL_CONF_KEY ="hbase.procedure.cleaner.acked.evict.ttl";<a name="line.92"></a>
+<span class="sourceLineNo">093</span> static final int DEFAULT_ACKED_EVICT_TTL = 5 * 60000; // 5min<a name="line.93"></a>
+<span class="sourceLineNo">094</span><a name="line.94"></a>
+<span class="sourceLineNo">095</span> /**<a name="line.95"></a>
+<span class="sourceLineNo">096</span> * {@link #testing} is non-null when ProcedureExecutor is being tested. Tests will try to<a name="line.96"></a>
+<span class="sourceLineNo">097</span> * break PE having it fail at various junctures. When non-null, testing is set to an instance of<a name="line.97"></a>
+<span class="sourceLineNo">098</span> * the below internal {@link Testing} class with flags set for the particular test.<a name="line.98"></a>
+<span class="sourceLineNo">099</span> */<a name="line.99"></a>
+<span class="sourceLineNo">100</span> volatile Testing testing = null;<a name="line.100"></a>
+<span class="sourceLineNo">101</span><a name="line.101"></a>
+<span class="sourceLineNo">102</span> /**<a name="line.102"></a>
+<span class="sourceLineNo">103</span> * Class with parameters describing how to fail/die when in testing-context.<a name="line.103"></a>
+<span class="sourceLineNo">104</span> */<a name="line.104"></a>
+<span class="sourceLineNo">105</span> public static class Testing {<a name="line.105"></a>
+<span class="sourceLineNo">106</span> protected volatile boolean killIfHasParent = true;<a name="line.106"></a>
+<span class="sourceLineNo">107</span> protected volatile boolean killIfSuspended = false;<a name="line.107"></a>
+<span class="sourceLineNo">108</span><a name="line.108"></a>
+<span class="sourceLineNo">109</span> /**<a name="line.109"></a>
+<span class="sourceLineNo">110</span> * Kill the PE BEFORE we store state to the WAL. Good for figuring out if a Procedure is<a name="line.110"></a>
+<span class="sourceLineNo">111</span> * persisting all the state it needs to recover after a crash.<a name="line.111"></a>
+<span class="sourceLineNo">112</span> */<a name="line.112"></a>
+<span class="sourceLineNo">113</span> protected volatile boolean killBeforeStoreUpdate = false;<a name="line.113"></a>
+<span class="sourceLineNo">114</span> protected volatile boolean toggleKillBeforeStoreUpdate = false;<a name="line.114"></a>
+<span class="sourceLineNo">115</span><a name="line.115"></a>
+<span class="sourceLineNo">116</span> /**<a name="line.116"></a>
+<span class="sourceLineNo">117</span> * Set when we want to fail AFTER state has been stored into the WAL. Rarely used. HBASE-20978<a name="line.117"></a>
+<span class="sourceLineNo">118</span> * is about a case where memory-state was being set after store to WAL where a crash could<a name="line.118"></a>
+<span class="sourceLineNo">119</span> * cause us to get stuck. This flag allows killing at what was a vulnerable time.<a name="line.119"></a>
+<span class="sourceLineNo">120</span> */<a name="line.120"></a>
+<span class="sourceLineNo">121</span> protected volatile boolean killAfterStoreUpdate = false;<a name="line.121"></a>
+<span class="sourceLineNo">122</span> protected volatile boolean toggleKillAfterStoreUpdate = false;<a name="line.122"></a>
+<span class="sourceLineNo">123</span><a name="line.123"></a>
+<span class="sourceLineNo">124</span> protected boolean shouldKillBeforeStoreUpdate() {<a name="line.124"></a>
+<span class="sourceLineNo">125</span> final boolean kill = this.killBeforeStoreUpdate;<a name="line.125"></a>
+<span class="sourceLineNo">126</span> if (this.toggleKillBeforeStoreUpdate) {<a name="line.126"></a>
+<span class="sourceLineNo">127</span> this.killBeforeStoreUpdate = !kill;<a name="line.127"></a>
+<span class="sourceLineNo">128</span> LOG.warn("Toggle KILL before store update to: " + this.killBeforeStoreUpdate);<a name="line.128"></a>
+<span class="sourceLineNo">129</span> }<a name="line.129"></a>
+<span class="sourceLineNo">130</span> return kill;<a name="line.130"></a>
+<span class="sourceLineNo">131</span> }<a name="line.131"></a>
+<span class="sourceLineNo">132</span><a name="line.132"></a>
+<span class="sourceLineNo">133</span> protected boolean shouldKillBeforeStoreUpdate(boolean isSuspended, boolean hasParent) {<a name="line.133"></a>
+<span class="sourceLineNo">134</span> if (isSuspended && !killIfSuspended) {<a name="line.134"></a>
+<span class="sourceLineNo">135</span> return false;<a name="line.135"></a>
+<span class="sourceLineNo">136</span> }<a name="line.136"></a>
+<span class="sourceLineNo">137</span> if (hasParent && !killIfHasParent) {<a name="line.137"></a>
+<span class="sourceLineNo">138</span> return false;<a name="line.138"></a>
+<span class="sourceLineNo">139</span> }<a name="line.139"></a>
+<span class="sourceLineNo">140</span> return shouldKillBeforeStoreUpdate();<a name="line.140"></a>
+<span class="sourceLineNo">141</span> }<a name="line.141"></a>
+<span class="sourceLineNo">142</span><a name="line.142"></a>
+<span class="sourceLineNo">143</span> protected boolean shouldKillAfterStoreUpdate() {<a name="line.143"></a>
+<span class="sourceLineNo">144</span> final boolean kill = this.killAfterStoreUpdate;<a name="line.144"></a>
+<span class="sourceLineNo">145</span> if (this.toggleKillAfterStoreUpdate) {<a name="line.145"></a>
+<span class="sourceLineNo">146</span> this.killAfterStoreUpdate = !kill;<a name="line.146"></a>
+<span class="sourceLineNo">147</span> LOG.warn("Toggle KILL after store update to: " + this.killAfterStoreUpdate);<a name="line.147"></a>
+<span class="sourceLineNo">148</span> }<a name="line.148"></a>
+<span class="sourceLineNo">149</span> return kill;<a name="line.149"></a>
+<span class="sourceLineNo">150</span> }<a name="line.150"></a>
+<span class="sourceLineNo">151</span><a name="line.151"></a>
+<span class="sourceLineNo">152</span> protected boolean shouldKillAfterStoreUpdate(final boolean isSuspended) {<a name="line.152"></a>
+<span class="sourceLineNo">153</span> return (isSuspended && !killIfSuspended) ? false : shouldKillAfterStoreUpdate();<a name="line.153"></a>
+<span class="sourceLineNo">154</span> }<a name="line.154"></a>
+<span class="sourceLineNo">155</span> }<a name="line.155"></a>
+<span class="sourceLineNo">156</span><a name="line.156"></a>
+<span class="sourceLineNo">157</span> public interface ProcedureExecutorListener {<a name="line.157"></a>
+<span class="sourceLineNo">158</span> void procedureLoaded(long procId);<a name="line.158"></a>
+<span class="sourceLineNo">159</span> void procedureAdded(long procId);<a name="line.159"></a>
+<span class="sourceLineNo">160</span> void procedureFinished(long procId);<a name="line.160"></a>
+<span class="sourceLineNo">161</span> }<a name="line.161"></a>
+<span class="sourceLineNo">162</span><a name="line.162"></a>
+<span class="sourceLineNo">163</span> /**<a name="line.163"></a>
+<span class="sourceLineNo">164</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the Procedure.<a name="line.164"></a>
+<span class="sourceLineNo">165</span> * Once a Root-Procedure completes (success or failure), the result will be added to this map.<a name="line.165"></a>
+<span class="sourceLineNo">166</span> * The user of ProcedureExecutor should call getResult(procId) to get the result.<a name="line.166"></a>
+<span class="sourceLineNo">167</span> */<a name="line.167"></a>
+<span class="sourceLineNo">168</span> private final ConcurrentHashMap<Long, CompletedProcedureRetainer<TEnvironment>> completed =<a name="line.168"></a>
+<span class="sourceLineNo">169</span> new ConcurrentHashMap<>();<a name="line.169"></a>
+<span class="sourceLineNo">170</span><a name="line.170"></a>
+<span class="sourceLineNo">171</span> /**<a name="line.171"></a>
+<span class="sourceLineNo">172</span> * Map the the procId returned by submitProcedure(), the Root-ProcID, to the RootProcedureState.<a name="line.172"></a>
+<span class="sourceLineNo">173</span> * The RootProcedureState contains the execution stack of the Root-Procedure,<a name="line.173"></a>
+<span class="sourceLineNo">174</span> * It is added to the map by submitProcedure() and removed on procedure completion.<a name="line.174"></a>
+<span class="sourceLineNo">175</span> */<a name="line.175"></a>
+<span class="sourceLineNo">176</span> private final ConcurrentHashMap<Long, RootProcedureState<TEnvironment>> rollbackStack =<a name="line.176"></a>
+<span class="sourceLineNo">177</span> new ConcurrentHashMap<>();<a name="line.177"></a>
+<span class="sourceLineNo">178</span><a name="line.178"></a>
+<span class="sourceLineNo">179</span> /**<a name="line.179"></a>
+<span class="sourceLineNo">180</span> * Helper map to lookup the live procedures by ID.<a name="line.180"></a>
+<span class="sourceLineNo">181</span> * This map contains every procedure. root-procedures and subprocedures.<a name="line.181"></a>
+<span class="sourceLineNo">182</span> */<a name="line.182"></a>
+<span class="sourceLineNo">183</span> private final ConcurrentHashMap<Long, Procedure<TEnvironment>> procedures =<a name="line.183"></a>
+<span class="sourceLineNo">184</span> new ConcurrentHashMap<>();<a name="line.184"></a>
+<span class="sourceLineNo">185</span><a name="line.185"></a>
+<span class="sourceLineNo">186</span> /**<a name="line.186"></a>
+<span class="sourceLineNo">187</span> * Helper map to lookup whether the procedure already issued from the same client. This map<a name="line.187"></a>
+<span class="sourceLineNo">188</span> * contains every root procedure.<a name="line.188"></a>
+<span class="sourceLineNo">189</span> */<a name="line.189"></a>
+<span class="sourceLineNo">190</span> private final ConcurrentHashMap<NonceKey, Long> nonceKeysToProcIdsMap = new ConcurrentHashMap<>();<a name="line.190"></a>
+<span class="sourceLineNo">191</span><a name="line.191"></a>
+<span class="sourceLineNo">192</span> private final CopyOnWriteArrayList<ProcedureExecutorListener> listeners =<a name="line.192"></a>
+<span class="sourceLineNo">193</span> new CopyOnWriteArrayList<>();<a name="line.193"></a>
+<span class="sourceLineNo">194</span><a name="line.194"></a>
+<span class="sourceLineNo">195</span> private Configuration conf;<a name="line.195"></a>
+<span class="sourceLineNo">196</span><a name="line.196"></a>
+<span class="sourceLineNo">197</span> /**<a name="line.197"></a>
+<span class="sourceLineNo">198</span> * Created in the {@link #init(int, boolean)} method. Destroyed in {@link #join()} (FIX! Doing<a name="line.198"></a>
+<span class="sourceLineNo">199</span> * resource handling rather than observing in a #join is unexpected).<a name="line.199"></a>
+<span class="sourceLineNo">200</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.200"></a>
+<span class="sourceLineNo">201</span> * (Should be ok).<a name="line.201"></a>
+<span class="sourceLineNo">202</span> */<a name="line.202"></a>
+<span class="sourceLineNo">203</span> private ThreadGroup threadGroup;<a name="line.203"></a>
+<span class="sourceLineNo">204</span><a name="line.204"></a>
+<span class="sourceLineNo">205</span> /**<a name="line.205"></a>
+<span class="sourceLineNo">206</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.206"></a>
+<span class="sourceLineNo">207</span> * resource handling rather than observing in a #join is unexpected).<a name="line.207"></a>
+<span class="sourceLineNo">208</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.208"></a>
+<span class="sourceLineNo">209</span> * (Should be ok).<a name="line.209"></a>
+<span class="sourceLineNo">210</span> */<a name="line.210"></a>
+<span class="sourceLineNo">211</span> private CopyOnWriteArrayList<WorkerThread> workerThreads;<a name="line.211"></a>
+<span class="sourceLineNo">212</span><a name="line.212"></a>
+<span class="sourceLineNo">213</span> /**<a name="line.213"></a>
+<span class="sourceLineNo">214</span> * Created in the {@link #init(int, boolean)} method. Terminated in {@link #join()} (FIX! Doing<a name="line.214"></a>
+<span class="sourceLineNo">215</span> * resource handling rather than observing in a #join is unexpected).<a name="line.215"></a>
+<span class="sourceLineNo">216</span> * Overridden when we do the ProcedureTestingUtility.testRecoveryAndDoubleExecution trickery<a name="line.216"></a>
+<span class="sourceLineNo">217</span> * (Should be ok).<a name="line.217"></a>
+<span class="sourceLineNo">218</span> */<a name="line.218"></a>
+<span class="sourceLineNo">219</span> private TimeoutExecutorThread<TEnvironment> timeoutExecutor;<a name="line.219"></a>
+<span class="sourceLineNo">220</span><a name="line.220"></a>
+<span class="sourceLineNo">221</span> /**<a name="line.221"></a>
+<span class="sourceLineNo">222</span> * WorkerMonitor check for stuck workers and new worker thread when necessary, for example if<a name="line.222"></a>
+<span class="sourceLineNo">223</span> * there is no worker to assign meta, it will new worker thread for it, so it is very important.<a name="line.223"></a>
+<span class="sourceLineNo">224</span> * TimeoutExecutor execute many tasks like DeadServerMetricRegionChore RegionInTransitionChore<a name="line.224"></a>
+<span class="sourceLineNo">225</span> * and so on, some tasks may execute for a long time so will block other tasks like<a name="line.225"></a>
+<span class="sourceLineNo">226</span> * WorkerMonitor, so use a dedicated thread for executing WorkerMonitor.<a name="line.226"></a>
+<span class="sourceLineNo">227</span> */<a name="line.227"></a>
+<span class="sourceLineNo">228</span> private TimeoutExecutorThread<TEnvironment> workerMonitorExecutor;<a name="line.228"></a>
+<span class="sourceLineNo">229</span><a name="line.229"></a>
+<span class="sourceLineNo">230</span> private int corePoolSize;<a name="line.230"></a>
+<span class="sourceLineNo">231</span> private int maxPoolSize;<a name="line.231"></a>
+<span class="sourceLineNo">232</span><a name="line.232"></a>
+<span class="sourceLineNo">233</span> private volatile long keepAliveTime;<a name="line.233"></a>
+<span class="sourceLineNo">234</span><a name="line.234"></a>
+<span class="sourceLineNo">235</span> /**<a name="line.235"></a>
+<span class="sourceLineNo">236</span> * Scheduler/Queue that contains runnable procedures.<a name="line.236"></a>
+<span class="sourceLineNo">237</span> */<a name="line.237"></a>
+<span class="sourceLineNo">238</span> private final ProcedureScheduler scheduler;<a name="line.238"></a>
+<span class="sourceLineNo">239</span><a name="line.239"></a>
+<span class="sourceLineNo">240</span> private final Executor forceUpdateExecutor = Executors.newSingleThreadExecutor(<a name="line.240"></a>
+<span class="sourceLineNo">241</span> new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Force-Update-PEWorker-%d").build());<a name="line.241"></a>
+<span class="sourceLineNo">242</span><a name="line.242"></a>
+<span class="sourceLineNo">243</span> private final AtomicLong lastProcId = new AtomicLong(-1);<a name="line.243"></a>
+<span class="sourceLineNo">244</span> private final AtomicLong workerId = new AtomicLong(0);<a name="line.244"></a>
+<span class="sourceLineNo">245</span> private final AtomicInteger activeExecutorCount = new AtomicInteger(0);<a name="line.245"></a>
+<span class="sourceLineNo">246</span> private final AtomicBoolean running = new AtomicBoolean(false);<a name="line.246"></a>
+<span class="sourceLineNo">247</span> private final TEnvironment environment;<a name="line.247"></a>
+<span class="sourceLineNo">248</span> private final ProcedureStore store;<a name="line.248"></a>
+<span class="sourceLineNo">249</span><a name="line.249"></a>
+<span class="sourceLineNo">250</span> private final boolean checkOwnerSet;<a name="line.250"></a>
+<span class="sourceLineNo">251</span><a name="line.251"></a>
+<span class="sourceLineNo">252</span> // To prevent concurrent execution of the same procedure.<a name="line.252"></a>
+<span class="sourceLineNo">253</span> // For some rare cases, especially if the procedure uses ProcedureEvent, it is possible that the<a name="line.253"></a>
+<span class="sourceLineNo">254</span> // procedure is woken up before we finish the suspend which causes the same procedures to be<a name="line.254"></a>
+<span class="sourceLineNo">255</span> // executed in parallel. This does lead to some problems, see HBASE-20939&HBASE-20949, and is also<a name="line.255"></a>
+<span class="sourceLineNo">256</span> // a bit confusing to the developers. So here we introduce this lock to prevent the concurrent<a name="line.256"></a>
+<span class="sourceLineNo">257</span> // execution of the same procedure.<a name="line.257"></a>
+<span class="sourceLineNo">258</span> private final IdLock procExecutionLock = new IdLock();<a name="line.258"></a>
+<span class="sourceLineNo">259</span><a name="line.259"></a>
+<span class="sourceLineNo">260</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.260"></a>
+<span class="sourceLineNo">261</span> final ProcedureStore store) {<a name="line.261"></a>
+<span class="sourceLineNo">262</span> this(conf, environment, store, new SimpleProcedureScheduler());<a name="line.262"></a>
+<span class="sourceLineNo">263</span> }<a name="line.263"></a>
+<span class="sourceLineNo">264</span><a name="line.264"></a>
+<span class="sourceLineNo">265</span> private boolean isRootFinished(Procedure<?> proc) {<a name="line.265"></a>
+<span class="sourceLineNo">266</span> Procedure<?> rootProc = procedures.get(proc.getRootProcId());<a name="line.266"></a>
+<span class="sourceLineNo">267</span> return rootProc == null || rootProc.isFinished();<a name="line.267"></a>
+<span class="sourceLineNo">268</span> }<a name="line.268"></a>
+<span class="sourceLineNo">269</span><a name="line.269"></a>
+<span class="sourceLineNo">270</span> private void forceUpdateProcedure(long procId) throws IOException {<a name="line.270"></a>
+<span class="sourceLineNo">271</span> IdLock.Entry lockEntry = procExecutionLock.getLockEntry(procId);<a name="line.271"></a>
+<span class="sourceLineNo">272</span> try {<a name="line.272"></a>
+<span class="sourceLineNo">273</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.273"></a>
+<span class="sourceLineNo">274</span> if (proc != null) {<a name="line.274"></a>
+<span class="sourceLineNo">275</span> if (proc.isFinished() && proc.hasParent() && isRootFinished(proc)) {<a name="line.275"></a>
+<span class="sourceLineNo">276</span> LOG.debug("Procedure {} has already been finished and parent is succeeded," +<a name="line.276"></a>
+<span class="sourceLineNo">277</span> " skip force updating", proc);<a name="line.277"></a>
+<span class="sourceLineNo">278</span> return;<a name="line.278"></a>
+<span class="sourceLineNo">279</span> }<a name="line.279"></a>
+<span class="sourceLineNo">280</span> } else {<a name="line.280"></a>
+<span class="sourceLineNo">281</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.281"></a>
+<span class="sourceLineNo">282</span> if (retainer == null || retainer.getProcedure() instanceof FailedProcedure) {<a name="line.282"></a>
+<span class="sourceLineNo">283</span> LOG.debug("No pending procedure with id = {}, skip force updating.", procId);<a name="line.283"></a>
+<span class="sourceLineNo">284</span> return;<a name="line.284"></a>
+<span class="sourceLineNo">285</span> }<a name="line.285"></a>
+<span class="sourceLineNo">286</span> long evictTtl = conf.getInt(EVICT_TTL_CONF_KEY, DEFAULT_EVICT_TTL);<a name="line.286"></a>
+<span class="sourceLineNo">287</span> long evictAckTtl = conf.getInt(EVICT_ACKED_TTL_CONF_KEY, DEFAULT_ACKED_EVICT_TTL);<a name="line.287"></a>
+<span class="sourceLineNo">288</span> if (retainer.isExpired(System.currentTimeMillis(), evictTtl, evictAckTtl)) {<a name="line.288"></a>
+<span class="sourceLineNo">289</span> LOG.debug("Procedure {} has already been finished and expired, skip force updating",<a name="line.289"></a>
+<span class="sourceLineNo">290</span> procId);<a name="line.290"></a>
+<span class="sourceLineNo">291</span> return;<a name="line.291"></a>
+<span class="sourceLineNo">292</span> }<a name="line.292"></a>
+<span class="sourceLineNo">293</span> proc = retainer.getProcedure();<a name="line.293"></a>
+<span class="sourceLineNo">294</span> }<a name="line.294"></a>
+<span class="sourceLineNo">295</span> LOG.debug("Force update procedure {}", proc);<a name="line.295"></a>
+<span class="sourceLineNo">296</span> store.update(proc);<a name="line.296"></a>
+<span class="sourceLineNo">297</span> } finally {<a name="line.297"></a>
+<span class="sourceLineNo">298</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.298"></a>
+<span class="sourceLineNo">299</span> }<a name="line.299"></a>
+<span class="sourceLineNo">300</span> }<a name="line.300"></a>
+<span class="sourceLineNo">301</span><a name="line.301"></a>
+<span class="sourceLineNo">302</span> public ProcedureExecutor(final Configuration conf, final TEnvironment environment,<a name="line.302"></a>
+<span class="sourceLineNo">303</span> final ProcedureStore store, final ProcedureScheduler scheduler) {<a name="line.303"></a>
+<span class="sourceLineNo">304</span> this.environment = environment;<a name="line.304"></a>
+<span class="sourceLineNo">305</span> this.scheduler = scheduler;<a name="line.305"></a>
+<span class="sourceLineNo">306</span> this.store = store;<a name="line.306"></a>
+<span class="sourceLineNo">307</span> this.conf = conf;<a name="line.307"></a>
+<span class="sourceLineNo">308</span> this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET);<a name="line.308"></a>
+<span class="sourceLineNo">309</span> refreshConfiguration(conf);<a name="line.309"></a>
+<span class="sourceLineNo">310</span> store.registerListener(new ProcedureStoreListener() {<a name="line.310"></a>
+<span class="sourceLineNo">311</span><a name="line.311"></a>
+<span class="sourceLineNo">312</span> @Override<a name="line.312"></a>
+<span class="sourceLineNo">313</span> public void forceUpdate(long[] procIds) {<a name="line.313"></a>
+<span class="sourceLineNo">314</span> Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> {<a name="line.314"></a>
+<span class="sourceLineNo">315</span> try {<a name="line.315"></a>
+<span class="sourceLineNo">316</span> forceUpdateProcedure(procId);<a name="line.316"></a>
+<span class="sourceLineNo">317</span> } catch (IOException e) {<a name="line.317"></a>
+<span class="sourceLineNo">318</span> LOG.warn("Failed to force update procedure with pid={}", procId);<a name="line.318"></a>
+<span class="sourceLineNo">319</span> }<a name="line.319"></a>
+<span class="sourceLineNo">320</span> }));<a name="line.320"></a>
+<span class="sourceLineNo">321</span> }<a name="line.321"></a>
+<span class="sourceLineNo">322</span> });<a name="line.322"></a>
+<span class="sourceLineNo">323</span> }<a name="line.323"></a>
+<span class="sourceLineNo">324</span><a name="line.324"></a>
+<span class="sourceLineNo">325</span> private void load(final boolean abortOnCorruption) throws IOException {<a name="line.325"></a>
+<span class="sourceLineNo">326</span> Preconditions.checkArgument(completed.isEmpty(), "completed not empty");<a name="line.326"></a>
+<span class="sourceLineNo">327</span> Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty");<a name="line.327"></a>
+<span class="sourceLineNo">328</span> Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty");<a name="line.328"></a>
+<span class="sourceLineNo">329</span> Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty");<a name="line.329"></a>
+<span class="sourceLineNo">330</span><a name="line.330"></a>
+<span class="sourceLineNo">331</span> store.load(new ProcedureStore.ProcedureLoader() {<a name="line.331"></a>
+<span class="sourceLineNo">332</span> @Override<a name="line.332"></a>
+<span class="sourceLineNo">333</span> public void setMaxProcId(long maxProcId) {<a name="line.333"></a>
+<span class="sourceLineNo">334</span> assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()";<a name="line.334"></a>
+<span class="sourceLineNo">335</span> lastProcId.set(maxProcId);<a name="line.335"></a>
+<span class="sourceLineNo">336</span> }<a name="line.336"></a>
+<span class="sourceLineNo">337</span><a name="line.337"></a>
+<span class="sourceLineNo">338</span> @Override<a name="line.338"></a>
+<span class="sourceLineNo">339</span> public void load(ProcedureIterator procIter) throws IOException {<a name="line.339"></a>
+<span class="sourceLineNo">340</span> loadProcedures(procIter, abortOnCorruption);<a name="line.340"></a>
+<span class="sourceLineNo">341</span> }<a name="line.341"></a>
+<span class="sourceLineNo">342</span><a name="line.342"></a>
+<span class="sourceLineNo">343</span> @Override<a name="line.343"></a>
+<span class="sourceLineNo">344</span> public void handleCorrupted(ProcedureIterator procIter) throws IOException {<a name="line.344"></a>
+<span class="sourceLineNo">345</span> int corruptedCount = 0;<a name="line.345"></a>
+<span class="sourceLineNo">346</span> while (procIter.hasNext()) {<a name="line.346"></a>
+<span class="sourceLineNo">347</span> Procedure<?> proc = procIter.next();<a name="line.347"></a>
+<span class="sourceLineNo">348</span> LOG.error("Corrupt " + proc);<a name="line.348"></a>
+<span class="sourceLineNo">349</span> corruptedCount++;<a name="line.349"></a>
+<span class="sourceLineNo">350</span> }<a name="line.350"></a>
+<span class="sourceLineNo">351</span> if (abortOnCorruption && corruptedCount > 0) {<a name="line.351"></a>
+<span class="sourceLineNo">352</span> throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay");<a name="line.352"></a>
+<span class="sourceLineNo">353</span> }<a name="line.353"></a>
+<span class="sourceLineNo">354</span> }<a name="line.354"></a>
+<span class="sourceLineNo">355</span> });<a name="line.355"></a>
+<span class="sourceLineNo">356</span> }<a name="line.356"></a>
+<span class="sourceLineNo">357</span><a name="line.357"></a>
+<span class="sourceLineNo">358</span> private void restoreLock(Procedure<TEnvironment> proc, Set<Long> restored) {<a name="line.358"></a>
+<span class="sourceLineNo">359</span> proc.restoreLock(getEnvironment());<a name="line.359"></a>
+<span class="sourceLineNo">360</span> restored.add(proc.getProcId());<a name="line.360"></a>
+<span class="sourceLineNo">361</span> }<a name="line.361"></a>
+<span class="sourceLineNo">362</span><a name="line.362"></a>
+<span class="sourceLineNo">363</span> private void restoreLocks(Deque<Procedure<TEnvironment>> stack, Set<Long> restored) {<a name="line.363"></a>
+<span class="sourceLineNo">364</span> while (!stack.isEmpty()) {<a name="line.364"></a>
+<span class="sourceLineNo">365</span> restoreLock(stack.pop(), restored);<a name="line.365"></a>
+<span class="sourceLineNo">366</span> }<a name="line.366"></a>
+<span class="sourceLineNo">367</span> }<a name="line.367"></a>
+<span class="sourceLineNo">368</span><a name="line.368"></a>
+<span class="sourceLineNo">369</span> // Restore the locks for all the procedures.<a name="line.369"></a>
+<span class="sourceLineNo">370</span> // Notice that we need to restore the locks starting from the root proc, otherwise there will be<a name="line.370"></a>
+<span class="sourceLineNo">371</span> // problem that a sub procedure may hold the exclusive lock first and then we are stuck when<a name="line.371"></a>
+<span class="sourceLineNo">372</span> // calling the acquireLock method for the parent procedure.<a name="line.372"></a>
+<span class="sourceLineNo">373</span> // The algorithm is straight-forward:<a name="line.373"></a>
+<span class="sourceLineNo">374</span> // 1. Use a set to record the procedures which locks have already been restored.<a name="line.374"></a>
+<span class="sourceLineNo">375</span> // 2. Use a stack to store the hierarchy of the procedures<a name="line.375"></a>
+<span class="sourceLineNo">376</span> // 3. For all the procedure, we will first try to find its parent and push it into the stack,<a name="line.376"></a>
+<span class="sourceLineNo">377</span> // unless<a name="line.377"></a>
+<span class="sourceLineNo">378</span> // a. We have no parent, i.e, we are the root procedure<a name="line.378"></a>
+<span class="sourceLineNo">379</span> // b. The lock has already been restored(by checking the set introduced in #1)<a name="line.379"></a>
+<span class="sourceLineNo">380</span> // then we start to pop the stack and call acquireLock for each procedure.<a name="line.380"></a>
+<span class="sourceLineNo">381</span> // Notice that this should be done for all procedures, not only the ones in runnableList.<a name="line.381"></a>
+<span class="sourceLineNo">382</span> private void restoreLocks() {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> Set<Long> restored = new HashSet<>();<a name="line.383"></a>
+<span class="sourceLineNo">384</span> Deque<Procedure<TEnvironment>> stack = new ArrayDeque<>();<a name="line.384"></a>
+<span class="sourceLineNo">385</span> procedures.values().forEach(proc -> {<a name="line.385"></a>
+<span class="sourceLineNo">386</span> for (;;) {<a name="line.386"></a>
+<span class="sourceLineNo">387</span> if (restored.contains(proc.getProcId())) {<a name="line.387"></a>
+<span class="sourceLineNo">388</span> restoreLocks(stack, restored);<a name="line.388"></a>
+<span class="sourceLineNo">389</span> return;<a name="line.389"></a>
+<span class="sourceLineNo">390</span> }<a name="line.390"></a>
+<span class="sourceLineNo">391</span> if (!proc.hasParent()) {<a name="line.391"></a>
+<span class="sourceLineNo">392</span> restoreLock(proc, restored);<a name="line.392"></a>
+<span class="sourceLineNo">393</span> restoreLocks(stack, restored);<a name="line.393"></a>
+<span class="sourceLineNo">394</span> return;<a name="line.394"></a>
+<span class="sourceLineNo">395</span> }<a name="line.395"></a>
+<span class="sourceLineNo">396</span> stack.push(proc);<a name="line.396"></a>
+<span class="sourceLineNo">397</span> proc = procedures.get(proc.getParentProcId());<a name="line.397"></a>
+<span class="sourceLineNo">398</span> }<a name="line.398"></a>
+<span class="sourceLineNo">399</span> });<a name="line.399"></a>
+<span class="sourceLineNo">400</span> }<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span> private void loadProcedures(ProcedureIterator procIter, boolean abortOnCorruption)<a name="line.402"></a>
+<span class="sourceLineNo">403</span> throws IOException {<a name="line.403"></a>
+<span class="sourceLineNo">404</span> // 1. Build the rollback stack<a name="line.404"></a>
+<span class="sourceLineNo">405</span> int runnableCount = 0;<a name="line.405"></a>
+<span class="sourceLineNo">406</span> int failedCount = 0;<a name="line.406"></a>
+<span class="sourceLineNo">407</span> int waitingCount = 0;<a name="line.407"></a>
+<span class="sourceLineNo">408</span> int waitingTimeoutCount = 0;<a name="line.408"></a>
+<span class="sourceLineNo">409</span> while (procIter.hasNext()) {<a name="line.409"></a>
+<span class="sourceLineNo">410</span> boolean finished = procIter.isNextFinished();<a name="line.410"></a>
+<span class="sourceLineNo">411</span> @SuppressWarnings("unchecked")<a name="line.411"></a>
+<span class="sourceLineNo">412</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.412"></a>
+<span class="sourceLineNo">413</span> NonceKey nonceKey = proc.getNonceKey();<a name="line.413"></a>
+<span class="sourceLineNo">414</span> long procId = proc.getProcId();<a name="line.414"></a>
+<span class="sourceLineNo">415</span><a name="line.415"></a>
+<span class="sourceLineNo">416</span> if (finished) {<a name="line.416"></a>
+<span class="sourceLineNo">417</span> completed.put(proc.getProcId(), new CompletedProcedureRetainer<>(proc));<a name="line.417"></a>
+<span class="sourceLineNo">418</span> LOG.debug("Completed {}", proc);<a name="line.418"></a>
+<span class="sourceLineNo">419</span> } else {<a name="line.419"></a>
+<span class="sourceLineNo">420</span> if (!proc.hasParent()) {<a name="line.420"></a>
+<span class="sourceLineNo">421</span> assert !proc.isFinished() : "unexpected finished procedure";<a name="line.421"></a>
+<span class="sourceLineNo">422</span> rollbackStack.put(proc.getProcId(), new RootProcedureState<>());<a name="line.422"></a>
+<span class="sourceLineNo">423</span> }<a name="line.423"></a>
+<span class="sourceLineNo">424</span><a name="line.424"></a>
+<span class="sourceLineNo">425</span> // add the procedure to the map<a name="line.425"></a>
+<span class="sourceLineNo">426</span> proc.beforeReplay(getEnvironment());<a name="line.426"></a>
+<span class="sourceLineNo">427</span> procedures.put(proc.getProcId(), proc);<a name="line.427"></a>
+<span class="sourceLineNo">428</span> switch (proc.getState()) {<a name="line.428"></a>
+<span class="sourceLineNo">429</span> case RUNNABLE:<a name="line.429"></a>
+<span class="sourceLineNo">430</span> runnableCount++;<a name="line.430"></a>
+<span class="sourceLineNo">431</span> break;<a name="line.431"></a>
+<span class="sourceLineNo">432</span> case FAILED:<a name="line.432"></a>
+<span class="sourceLineNo">433</span> failedCount++;<a name="line.433"></a>
+<span class="sourceLineNo">434</span> break;<a name="line.434"></a>
+<span class="sourceLineNo">435</span> case WAITING:<a name="line.435"></a>
+<span class="sourceLineNo">436</span> waitingCount++;<a name="line.436"></a>
+<span class="sourceLineNo">437</span> break;<a name="line.437"></a>
+<span class="sourceLineNo">438</span> case WAITING_TIMEOUT:<a name="line.438"></a>
+<span class="sourceLineNo">439</span> waitingTimeoutCount++;<a name="line.439"></a>
+<span class="sourceLineNo">440</span> break;<a name="line.440"></a>
+<span class="sourceLineNo">441</span> default:<a name="line.441"></a>
+<span class="sourceLineNo">442</span> break;<a name="line.442"></a>
+<span class="sourceLineNo">443</span> }<a name="line.443"></a>
+<span class="sourceLineNo">444</span> }<a name="line.444"></a>
+<span class="sourceLineNo">445</span><a name="line.445"></a>
+<span class="sourceLineNo">446</span> if (nonceKey != null) {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> nonceKeysToProcIdsMap.put(nonceKey, procId); // add the nonce to the map<a name="line.447"></a>
+<span class="sourceLineNo">448</span> }<a name="line.448"></a>
+<span class="sourceLineNo">449</span> }<a name="line.449"></a>
+<span class="sourceLineNo">450</span><a name="line.450"></a>
+<span class="sourceLineNo">451</span> // 2. Initialize the stacks: In the old implementation, for procedures in FAILED state, we will<a name="line.451"></a>
+<span class="sourceLineNo">452</span> // push it into the ProcedureScheduler directly to execute the rollback. But this does not work<a name="line.452"></a>
+<span class="sourceLineNo">453</span> // after we introduce the restore lock stage. For now, when we acquire a xlock, we will remove<a name="line.453"></a>
+<span class="sourceLineNo">454</span> // the queue from runQueue in scheduler, and then when a procedure which has lock access, for<a name="line.454"></a>
+<span class="sourceLineNo">455</span> // example, a sub procedure of the procedure which has the xlock, is pushed into the scheduler,<a name="line.455"></a>
+<span class="sourceLineNo">456</span> // we will add the queue back to let the workers poll from it. The assumption here is that, the<a name="line.456"></a>
+<span class="sourceLineNo">457</span> // procedure which has the xlock should have been polled out already, so when loading we can not<a name="line.457"></a>
+<span class="sourceLineNo">458</span> // add the procedure to scheduler first and then call acquireLock, since the procedure is still<a name="line.458"></a>
+<span class="sourceLineNo">459</span> // in the queue, and since we will remove the queue from runQueue, then no one can poll it out,<a name="line.459"></a>
+<span class="sourceLineNo">460</span> // then there is a dead lock<a name="line.460"></a>
+<span class="sourceLineNo">461</span> List<Procedure<TEnvironment>> runnableList = new ArrayList<>(runnableCount);<a name="line.461"></a>
+<span class="sourceLineNo">462</span> List<Procedure<TEnvironment>> failedList = new ArrayList<>(failedCount);<a name="line.462"></a>
+<span class="sourceLineNo">463</span> List<Procedure<TEnvironment>> waitingList = new ArrayList<>(waitingCount);<a name="line.463"></a>
+<span class="sourceLineNo">464</span> List<Procedure<TEnvironment>> waitingTimeoutList = new ArrayList<>(waitingTimeoutCount);<a name="line.464"></a>
+<span class="sourceLineNo">465</span> procIter.reset();<a name="line.465"></a>
+<span class="sourceLineNo">466</span> while (procIter.hasNext()) {<a name="line.466"></a>
+<span class="sourceLineNo">467</span> if (procIter.isNextFinished()) {<a name="line.467"></a>
+<span class="sourceLineNo">468</span> procIter.skipNext();<a name="line.468"></a>
+<span class="sourceLineNo">469</span> continue;<a name="line.469"></a>
+<span class="sourceLineNo">470</span> }<a name="line.470"></a>
+<span class="sourceLineNo">471</span><a name="line.471"></a>
+<span class="sourceLineNo">472</span> @SuppressWarnings("unchecked")<a name="line.472"></a>
+<span class="sourceLineNo">473</span> Procedure<TEnvironment> proc = procIter.next();<a name="line.473"></a>
+<span class="sourceLineNo">474</span> assert !(proc.isFinished() && !proc.hasParent()) : "unexpected completed proc=" + proc;<a name="line.474"></a>
+<span class="sourceLineNo">475</span> LOG.debug("Loading {}", proc);<a name="line.475"></a>
+<span class="sourceLineNo">476</span> Long rootProcId = getRootProcedureId(proc);<a name="line.476"></a>
+<span class="sourceLineNo">477</span> // The orphan procedures will be passed to handleCorrupted, so add an assert here<a name="line.477"></a>
+<span class="sourceLineNo">478</span> assert rootProcId != null;<a name="line.478"></a>
+<span class="sourceLineNo">479</span><a name="line.479"></a>
+<span class="sourceLineNo">480</span> if (proc.hasParent()) {<a name="line.480"></a>
+<span class="sourceLineNo">481</span> Procedure<TEnvironment> parent = procedures.get(proc.getParentProcId());<a name="line.481"></a>
+<span class="sourceLineNo">482</span> if (parent != null && !proc.isFinished()) {<a name="line.482"></a>
+<span class="sourceLineNo">483</span> parent.incChildrenLatch();<a name="line.483"></a>
+<span class="sourceLineNo">484</span> }<a name="line.484"></a>
+<span class="sourceLineNo">485</span> }<a name="line.485"></a>
+<span class="sourceLineNo">486</span><a name="line.486"></a>
+<span class="sourceLineNo">487</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.487"></a>
+<span class="sourceLineNo">488</span> procStack.loadStack(proc);<a name="line.488"></a>
+<span class="sourceLineNo">489</span><a name="line.489"></a>
+<span class="sourceLineNo">490</span> proc.setRootProcId(rootProcId);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> switch (proc.getState()) {<a name="line.491"></a>
+<span class="sourceLineNo">492</span> case RUNNABLE:<a name="line.492"></a>
+<span class="sourceLineNo">493</span> runnableList.add(proc);<a name="line.493"></a>
+<span class="sourceLineNo">494</span> break;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> case WAITING:<a name="line.495"></a>
+<span class="sourceLineNo">496</span> waitingList.add(proc);<a name="line.496"></a>
+<span class="sourceLineNo">497</span> break;<a name="line.497"></a>
+<span class="sourceLineNo">498</span> case WAITING_TIMEOUT:<a name="line.498"></a>
+<span class="sourceLineNo">499</span> waitingTimeoutList.add(proc);<a name="line.499"></a>
+<span class="sourceLineNo">500</span> break;<a name="line.500"></a>
+<span class="sourceLineNo">501</span> case FAILED:<a name="line.501"></a>
+<span class="sourceLineNo">502</span> failedList.add(proc);<a name="line.502"></a>
+<span class="sourceLineNo">503</span> break;<a name="line.503"></a>
+<span class="sourceLineNo">504</span> case ROLLEDBACK:<a name="line.504"></a>
+<span class="sourceLineNo">505</span> case INITIALIZING:<a name="line.505"></a>
+<span class="sourceLineNo">506</span> String msg = "Unexpected " + proc.getState() + " state for " + proc;<a name="line.506"></a>
+<span class="sourceLineNo">507</span> LOG.error(msg);<a name="line.507"></a>
+<span class="sourceLineNo">508</span> throw new UnsupportedOperationException(msg);<a name="line.508"></a>
+<span class="sourceLineNo">509</span> default:<a name="line.509"></a>
+<span class="sourceLineNo">510</span> break;<a name="line.510"></a>
+<span class="sourceLineNo">511</span> }<a name="line.511"></a>
+<span class="sourceLineNo">512</span> }<a name="line.512"></a>
+<span class="sourceLineNo">513</span><a name="line.513"></a>
+<span class="sourceLineNo">514</span> // 3. Check the waiting procedures to see if some of them can be added to runnable.<a name="line.514"></a>
+<span class="sourceLineNo">515</span> waitingList.forEach(proc -> {<a name="line.515"></a>
+<span class="sourceLineNo">516</span> if (!proc.hasChildren()) {<a name="line.516"></a>
+<span class="sourceLineNo">517</span> // Normally, WAITING procedures should be waken by its children. But, there is a case that,<a name="line.517"></a>
+<span class="sourceLineNo">518</span> // all the children are successful and before they can wake up their parent procedure, the<a name="line.518"></a>
+<span class="sourceLineNo">519</span> // master was killed. So, during recovering the procedures from ProcedureWal, its children<a name="line.519"></a>
+<span class="sourceLineNo">520</span> // are not loaded because of their SUCCESS state. So we need to continue to run this WAITING<a name="line.520"></a>
+<span class="sourceLineNo">521</span> // procedure. But before executing, we need to set its state to RUNNABLE, otherwise, a<a name="line.521"></a>
+<span class="sourceLineNo">522</span> // exception will throw:<a name="line.522"></a>
+<span class="sourceLineNo">523</span> // Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.523"></a>
+<span class="sourceLineNo">524</span> // "NOT RUNNABLE! " + procedure.toString());<a name="line.524"></a>
+<span class="sourceLineNo">525</span> proc.setState(ProcedureState.RUNNABLE);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> runnableList.add(proc);<a name="line.526"></a>
+<span class="sourceLineNo">527</span> } else {<a name="line.527"></a>
+<span class="sourceLineNo">528</span> proc.afterReplay(getEnvironment());<a name="line.528"></a>
+<span class="sourceLineNo">529</span> }<a name="line.529"></a>
+<span class="sourceLineNo">530</span> });<a name="line.530"></a>
+<span class="sourceLineNo">531</span> // 4. restore locks<a name="line.531"></a>
+<span class="sourceLineNo">532</span> restoreLocks();<a name="line.532"></a>
+<span class="sourceLineNo">533</span><a name="line.533"></a>
+<span class="sourceLineNo">534</span> // 5. Push the procedures to the timeout executor<a name="line.534"></a>
+<span class="sourceLineNo">535</span> waitingTimeoutList.forEach(proc -> {<a name="line.535"></a>
+<span class="sourceLineNo">536</span> proc.afterReplay(getEnvironment());<a name="line.536"></a>
+<span class="sourceLineNo">537</span> timeoutExecutor.add(proc);<a name="line.537"></a>
+<span class="sourceLineNo">538</span> });<a name="line.538"></a>
+<span class="sourceLineNo">539</span><a name="line.539"></a>
+<span class="sourceLineNo">540</span> // 6. Push the procedure to the scheduler<a name="line.540"></a>
+<span class="sourceLineNo">541</span> failedList.forEach(scheduler::addBack);<a name="line.541"></a>
+<span class="sourceLineNo">542</span> runnableList.forEach(p -> {<a name="line.542"></a>
+<span class="sourceLineNo">543</span> p.afterReplay(getEnvironment());<a name="line.543"></a>
+<span class="sourceLineNo">544</span> if (!p.hasParent()) {<a name="line.544"></a>
+<span class="sourceLineNo">545</span> sendProcedureLoadedNotification(p.getProcId());<a name="line.545"></a>
+<span class="sourceLineNo">546</span> }<a name="line.546"></a>
+<span class="sourceLineNo">547</span> scheduler.addBack(p);<a name="line.547"></a>
+<span class="sourceLineNo">548</span> });<a name="line.548"></a>
+<span class="sourceLineNo">549</span> // After all procedures put into the queue, signal the worker threads.<a name="line.549"></a>
+<span class="sourceLineNo">550</span> // Otherwise, there is a race condition. See HBASE-21364.<a name="line.550"></a>
+<span class="sourceLineNo">551</span> scheduler.signalAll();<a name="line.551"></a>
+<span class="sourceLineNo">552</span> }<a name="line.552"></a>
+<span class="sourceLineNo">553</span><a name="line.553"></a>
+<span class="sourceLineNo">554</span> /**<a name="line.554"></a>
+<span class="sourceLineNo">555</span> * Initialize the procedure executor, but do not start workers. We will start them later.<a name="line.555"></a>
+<span class="sourceLineNo">556</span> * <p/><a name="line.556"></a>
+<span class="sourceLineNo">557</span> * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to recover the lease, and<a name="line.557"></a>
+<span class="sourceLineNo">558</span> * ensure a single executor, and start the procedure replay to resume and recover the previous<a name="line.558"></a>
+<span class="sourceLineNo">559</span> * pending and in-progress procedures.<a name="line.559"></a>
+<span class="sourceLineNo">560</span> * @param numThreads number of threads available for procedure execution.<a name="line.560"></a>
+<span class="sourceLineNo">561</span> * @param abortOnCorruption true if you want to abort your service in case a corrupted procedure<a name="line.561"></a>
+<span class="sourceLineNo">562</span> * is found on replay. otherwise false.<a name="line.562"></a>
+<span class="sourceLineNo">563</span> */<a name="line.563"></a>
+<span class="sourceLineNo">564</span> public void init(int numThreads, boolean abortOnCorruption) throws IOException {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> // We have numThreads executor + one timer thread used for timing out<a name="line.565"></a>
+<span class="sourceLineNo">566</span> // procedures and triggering periodic procedures.<a name="line.566"></a>
+<span class="sourceLineNo">567</span> this.corePoolSize = numThreads;<a name="line.567"></a>
+<span class="sourceLineNo">568</span> this.maxPoolSize = 10 * numThreads;<a name="line.568"></a>
+<span class="sourceLineNo">569</span> LOG.info("Starting {} core workers (bigger of cpus/4 or 16) with max (burst) worker count={}",<a name="line.569"></a>
+<span class="sourceLineNo">570</span> corePoolSize, maxPoolSize);<a name="line.570"></a>
+<span class="sourceLineNo">571</span><a name="line.571"></a>
+<span class="sourceLineNo">572</span> this.threadGroup = new ThreadGroup("PEWorkerGroup");<a name="line.572"></a>
+<span class="sourceLineNo">573</span> this.timeoutExecutor = new TimeoutExecutorThread<>(this, threadGroup, "ProcExecTimeout");<a name="line.573"></a>
+<span class="sourceLineNo">574</span> this.workerMonitorExecutor = new TimeoutExecutorThread<>(this, threadGroup, "WorkerMonitor");<a name="line.574"></a>
+<span class="sourceLineNo">575</span><a name="line.575"></a>
+<span class="sourceLineNo">576</span> // Create the workers<a name="line.576"></a>
+<span class="sourceLineNo">577</span> workerId.set(0);<a name="line.577"></a>
+<span class="sourceLineNo">578</span> workerThreads = new CopyOnWriteArrayList<>();<a name="line.578"></a>
+<span class="sourceLineNo">579</span> for (int i = 0; i < corePoolSize; ++i) {<a name="line.579"></a>
+<span class="sourceLineNo">580</span> workerThreads.add(new WorkerThread(threadGroup));<a name="line.580"></a>
+<span class="sourceLineNo">581</span> }<a name="line.581"></a>
+<span class="sourceLineNo">582</span><a name="line.582"></a>
+<span class="sourceLineNo">583</span> long st, et;<a name="line.583"></a>
+<span class="sourceLineNo">584</span><a name="line.584"></a>
+<span class="sourceLineNo">585</span> // Acquire the store lease.<a name="line.585"></a>
+<span class="sourceLineNo">586</span> st = System.nanoTime();<a name="line.586"></a>
+<span class="sourceLineNo">587</span> store.recoverLease();<a name="line.587"></a>
+<span class="sourceLineNo">588</span> et = System.nanoTime();<a name="line.588"></a>
+<span class="sourceLineNo">589</span> LOG.info("Recovered {} lease in {}", store.getClass().getSimpleName(),<a name="line.589"></a>
+<span class="sourceLineNo">590</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.590"></a>
+<span class="sourceLineNo">591</span><a name="line.591"></a>
+<span class="sourceLineNo">592</span> // start the procedure scheduler<a name="line.592"></a>
+<span class="sourceLineNo">593</span> scheduler.start();<a name="line.593"></a>
+<span class="sourceLineNo">594</span><a name="line.594"></a>
+<span class="sourceLineNo">595</span> // TODO: Split in two steps.<a name="line.595"></a>
+<span class="sourceLineNo">596</span> // TODO: Handle corrupted procedures (currently just a warn)<a name="line.596"></a>
+<span class="sourceLineNo">597</span> // The first one will make sure that we have the latest id,<a name="line.597"></a>
+<span class="sourceLineNo">598</span> // so we can start the threads and accept new procedures.<a name="line.598"></a>
+<span class="sourceLineNo">599</span> // The second step will do the actual load of old procedures.<a name="line.599"></a>
+<span class="sourceLineNo">600</span> st = System.nanoTime();<a name="line.600"></a>
+<span class="sourceLineNo">601</span> load(abortOnCorruption);<a name="line.601"></a>
+<span class="sourceLineNo">602</span> et = System.nanoTime();<a name="line.602"></a>
+<span class="sourceLineNo">603</span> LOG.info("Loaded {} in {}", store.getClass().getSimpleName(),<a name="line.603"></a>
+<span class="sourceLineNo">604</span> StringUtils.humanTimeDiff(TimeUnit.NANOSECONDS.toMillis(et - st)));<a name="line.604"></a>
+<span class="sourceLineNo">605</span> }<a name="line.605"></a>
+<span class="sourceLineNo">606</span><a name="line.606"></a>
+<span class="sourceLineNo">607</span> /**<a name="line.607"></a>
+<span class="sourceLineNo">608</span> * Start the workers.<a name="line.608"></a>
+<span class="sourceLineNo">609</span> */<a name="line.609"></a>
+<span class="sourceLineNo">610</span> public void startWorkers() throws IOException {<a name="line.610"></a>
+<span class="sourceLineNo">611</span> if (!running.compareAndSet(false, true)) {<a name="line.611"></a>
+<span class="sourceLineNo">612</span> LOG.warn("Already running");<a name="line.612"></a>
+<span class="sourceLineNo">613</span> return;<a name="line.613"></a>
+<span class="sourceLineNo">614</span> }<a name="line.614"></a>
+<span class="sourceLineNo">615</span> // Start the executors. Here we must have the lastProcId set.<a name="line.615"></a>
+<span class="sourceLineNo">616</span> LOG.trace("Start workers {}", workerThreads.size());<a name="line.616"></a>
+<span class="sourceLineNo">617</span> timeoutExecutor.start();<a name="line.617"></a>
+<span class="sourceLineNo">618</span> workerMonitorExecutor.start();<a name="line.618"></a>
+<span class="sourceLineNo">619</span> for (WorkerThread worker: workerThreads) {<a name="line.619"></a>
+<span class="sourceLineNo">620</span> worker.start();<a name="line.620"></a>
+<span class="sourceLineNo">621</span> }<a name="line.621"></a>
+<span class="sourceLineNo">622</span><a name="line.622"></a>
+<span class="sourceLineNo">623</span> // Internal chores<a name="line.623"></a>
+<span class="sourceLineNo">624</span> workerMonitorExecutor.add(new WorkerMonitor());<a name="line.624"></a>
+<span class="sourceLineNo">625</span><a name="line.625"></a>
+<span class="sourceLineNo">626</span> // Add completed cleaner chore<a name="line.626"></a>
+<span class="sourceLineNo">627</span> addChore(new CompletedProcedureCleaner<>(conf, store, procExecutionLock, completed,<a name="line.627"></a>
+<span class="sourceLineNo">628</span> nonceKeysToProcIdsMap));<a name="line.628"></a>
+<span class="sourceLineNo">629</span> }<a name="line.629"></a>
+<span class="sourceLineNo">630</span><a name="line.630"></a>
+<span class="sourceLineNo">631</span> public void stop() {<a name="line.631"></a>
+<span class="sourceLineNo">632</span> if (!running.getAndSet(false)) {<a name="line.632"></a>
+<span class="sourceLineNo">633</span> return;<a name="line.633"></a>
+<span class="sourceLineNo">634</span> }<a name="line.634"></a>
+<span class="sourceLineNo">635</span><a name="line.635"></a>
+<span class="sourceLineNo">636</span> LOG.info("Stopping");<a name="line.636"></a>
+<span class="sourceLineNo">637</span> scheduler.stop();<a name="line.637"></a>
+<span class="sourceLineNo">638</span> timeoutExecutor.sendStopSignal();<a name="line.638"></a>
+<span class="sourceLineNo">639</span> workerMonitorExecutor.sendStopSignal();<a name="line.639"></a>
+<span class="sourceLineNo">640</span> }<a name="line.640"></a>
+<span class="sourceLineNo">641</span><a name="line.641"></a>
+<span class="sourceLineNo">642</span> @VisibleForTesting<a name="line.642"></a>
+<span class="sourceLineNo">643</span> public void join() {<a name="line.643"></a>
+<span class="sourceLineNo">644</span> assert !isRunning() : "expected not running";<a name="line.644"></a>
+<span class="sourceLineNo">645</span><a name="line.645"></a>
+<span class="sourceLineNo">646</span> // stop the timeout executor<a name="line.646"></a>
+<span class="sourceLineNo">647</span> timeoutExecutor.awaitTermination();<a name="line.647"></a>
+<span class="sourceLineNo">648</span> // stop the work monitor executor<a name="line.648"></a>
+<span class="sourceLineNo">649</span> workerMonitorExecutor.awaitTermination();<a name="line.649"></a>
+<span class="sourceLineNo">650</span><a name="line.650"></a>
+<span class="sourceLineNo">651</span> // stop the worker threads<a name="line.651"></a>
+<span class="sourceLineNo">652</span> for (WorkerThread worker: workerThreads) {<a name="line.652"></a>
+<span class="sourceLineNo">653</span> worker.awaitTermination();<a name="line.653"></a>
+<span class="sourceLineNo">654</span> }<a name="line.654"></a>
+<span class="sourceLineNo">655</span><a name="line.655"></a>
+<span class="sourceLineNo">656</span> // Destroy the Thread Group for the executors<a name="line.656"></a>
+<span class="sourceLineNo">657</span> // TODO: Fix. #join is not place to destroy resources.<a name="line.657"></a>
+<span class="sourceLineNo">658</span> try {<a name="line.658"></a>
+<span class="sourceLineNo">659</span> threadGroup.destroy();<a name="line.659"></a>
+<span class="sourceLineNo">660</span> } catch (IllegalThreadStateException e) {<a name="line.660"></a>
+<span class="sourceLineNo">661</span> LOG.error("ThreadGroup {} contains running threads; {}: See STDOUT",<a name="line.661"></a>
+<span class="sourceLineNo">662</span> this.threadGroup, e.getMessage());<a name="line.662"></a>
+<span class="sourceLineNo">663</span> // This dumps list of threads on STDOUT.<a name="line.663"></a>
+<span class="sourceLineNo">664</span> this.threadGroup.list();<a name="line.664"></a>
+<span class="sourceLineNo">665</span> }<a name="line.665"></a>
+<span class="sourceLineNo">666</span><a name="line.666"></a>
+<span class="sourceLineNo">667</span> // reset the in-memory state for testing<a name="line.667"></a>
+<span class="sourceLineNo">668</span> completed.clear();<a name="line.668"></a>
+<span class="sourceLineNo">669</span> rollbackStack.clear();<a name="line.669"></a>
+<span class="sourceLineNo">670</span> procedures.clear();<a name="line.670"></a>
+<span class="sourceLineNo">671</span> nonceKeysToProcIdsMap.clear();<a name="line.671"></a>
+<span class="sourceLineNo">672</span> scheduler.clear();<a name="line.672"></a>
+<span class="sourceLineNo">673</span> lastProcId.set(-1);<a name="line.673"></a>
+<span class="sourceLineNo">674</span> }<a name="line.674"></a>
+<span class="sourceLineNo">675</span><a name="line.675"></a>
+<span class="sourceLineNo">676</span> public void refreshConfiguration(final Configuration conf) {<a name="line.676"></a>
+<span class="sourceLineNo">677</span> this.conf = conf;<a name="line.677"></a>
+<span class="sourceLineNo">678</span> setKeepAliveTime(conf.getLong(WORKER_KEEP_ALIVE_TIME_CONF_KEY,<a name="line.678"></a>
+<span class="sourceLineNo">679</span> DEFAULT_WORKER_KEEP_ALIVE_TIME), TimeUnit.MILLISECONDS);<a name="line.679"></a>
+<span class="sourceLineNo">680</span> }<a name="line.680"></a>
+<span class="sourceLineNo">681</span><a name="line.681"></a>
+<span class="sourceLineNo">682</span> // ==========================================================================<a name="line.682"></a>
+<span class="sourceLineNo">683</span> // Accessors<a name="line.683"></a>
+<span class="sourceLineNo">684</span> // ==========================================================================<a name="line.684"></a>
+<span class="sourceLineNo">685</span> public boolean isRunning() {<a name="line.685"></a>
+<span class="sourceLineNo">686</span> return running.get();<a name="line.686"></a>
+<span class="sourceLineNo">687</span> }<a name="line.687"></a>
+<span class="sourceLineNo">688</span><a name="line.688"></a>
+<span class="sourceLineNo">689</span> /**<a name="line.689"></a>
+<span class="sourceLineNo">690</span> * @return the current number of worker threads.<a name="line.690"></a>
+<span class="sourceLineNo">691</span> */<a name="line.691"></a>
+<span class="sourceLineNo">692</span> public int getWorkerThreadCount() {<a name="line.692"></a>
+<span class="sourceLineNo">693</span> return workerThreads.size();<a name="line.693"></a>
+<span class="sourceLineNo">694</span> }<a name="line.694"></a>
+<span class="sourceLineNo">695</span><a name="line.695"></a>
+<span class="sourceLineNo">696</span> /**<a name="line.696"></a>
+<span class="sourceLineNo">697</span> * @return the core pool size settings.<a name="line.697"></a>
+<span class="sourceLineNo">698</span> */<a name="line.698"></a>
+<span class="sourceLineNo">699</span> public int getCorePoolSize() {<a name="line.699"></a>
+<span class="sourceLineNo">700</span> return corePoolSize;<a name="line.700"></a>
+<span class="sourceLineNo">701</span> }<a name="line.701"></a>
+<span class="sourceLineNo">702</span><a name="line.702"></a>
+<span class="sourceLineNo">703</span> public int getActiveExecutorCount() {<a name="line.703"></a>
+<span class="sourceLineNo">704</span> return activeExecutorCount.get();<a name="line.704"></a>
+<span class="sourceLineNo">705</span> }<a name="line.705"></a>
+<span class="sourceLineNo">706</span><a name="line.706"></a>
+<span class="sourceLineNo">707</span> public TEnvironment getEnvironment() {<a name="line.707"></a>
+<span class="sourceLineNo">708</span> return this.environment;<a name="line.708"></a>
+<span class="sourceLineNo">709</span> }<a name="line.709"></a>
+<span class="sourceLineNo">710</span><a name="line.710"></a>
+<span class="sourceLineNo">711</span> public ProcedureStore getStore() {<a name="line.711"></a>
+<span class="sourceLineNo">712</span> return this.store;<a name="line.712"></a>
+<span class="sourceLineNo">713</span> }<a name="line.713"></a>
+<span class="sourceLineNo">714</span><a name="line.714"></a>
+<span class="sourceLineNo">715</span> ProcedureScheduler getScheduler() {<a name="line.715"></a>
+<span class="sourceLineNo">716</span> return scheduler;<a name="line.716"></a>
+<span class="sourceLineNo">717</span> }<a name="line.717"></a>
+<span class="sourceLineNo">718</span><a name="line.718"></a>
+<span class="sourceLineNo">719</span> public void setKeepAliveTime(final long keepAliveTime, final TimeUnit timeUnit) {<a name="line.719"></a>
+<span class="sourceLineNo">720</span> this.keepAliveTime = timeUnit.toMillis(keepAliveTime);<a name="line.720"></a>
+<span class="sourceLineNo">721</span> this.scheduler.signalAll();<a name="line.721"></a>
+<span class="sourceLineNo">722</span> }<a name="line.722"></a>
+<span class="sourceLineNo">723</span><a name="line.723"></a>
+<span class="sourceLineNo">724</span> public long getKeepAliveTime(final TimeUnit timeUnit) {<a name="line.724"></a>
+<span class="sourceLineNo">725</span> return timeUnit.convert(keepAliveTime, TimeUnit.MILLISECONDS);<a name="line.725"></a>
+<span class="sourceLineNo">726</span> }<a name="line.726"></a>
+<span class="sourceLineNo">727</span><a name="line.727"></a>
+<span class="sourceLineNo">728</span> // ==========================================================================<a name="line.728"></a>
+<span class="sourceLineNo">729</span> // Submit/Remove Chores<a name="line.729"></a>
+<span class="sourceLineNo">730</span> // ==========================================================================<a name="line.730"></a>
+<span class="sourceLineNo">731</span><a name="line.731"></a>
+<span class="sourceLineNo">732</span> /**<a name="line.732"></a>
+<span class="sourceLineNo">733</span> * Add a chore procedure to the executor<a name="line.733"></a>
+<span class="sourceLineNo">734</span> * @param chore the chore to add<a name="line.734"></a>
+<span class="sourceLineNo">735</span> */<a name="line.735"></a>
+<span class="sourceLineNo">736</span> public void addChore(@Nullable ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.736"></a>
+<span class="sourceLineNo">737</span> if (chore == null) {<a name="line.737"></a>
+<span class="sourceLineNo">738</span> return;<a name="line.738"></a>
+<span class="sourceLineNo">739</span> }<a name="line.739"></a>
+<span class="sourceLineNo">740</span> chore.setState(ProcedureState.WAITING_TIMEOUT);<a name="line.740"></a>
+<span class="sourceLineNo">741</span> timeoutExecutor.add(chore);<a name="line.741"></a>
+<span class="sourceLineNo">742</span> }<a name="line.742"></a>
+<span class="sourceLineNo">743</span><a name="line.743"></a>
+<span class="sourceLineNo">744</span> /**<a name="line.744"></a>
+<span class="sourceLineNo">745</span> * Remove a chore procedure from the executor<a name="line.745"></a>
+<span class="sourceLineNo">746</span> * @param chore the chore to remove<a name="line.746"></a>
+<span class="sourceLineNo">747</span> * @return whether the chore is removed, or it will be removed later<a name="line.747"></a>
+<span class="sourceLineNo">748</span> */<a name="line.748"></a>
+<span class="sourceLineNo">749</span> public boolean removeChore(@Nullable ProcedureInMemoryChore<TEnvironment> chore) {<a name="line.749"></a>
+<span class="sourceLineNo">750</span> if (chore == null) {<a name="line.750"></a>
+<span class="sourceLineNo">751</span> return true;<a name="line.751"></a>
+<span class="sourceLineNo">752</span> }<a name="line.752"></a>
+<span class="sourceLineNo">753</span> chore.setState(ProcedureState.SUCCESS);<a name="line.753"></a>
+<span class="sourceLineNo">754</span> return timeoutExecutor.remove(chore);<a name="line.754"></a>
+<span class="sourceLineNo">755</span> }<a name="line.755"></a>
+<span class="sourceLineNo">756</span><a name="line.756"></a>
+<span class="sourceLineNo">757</span> // ==========================================================================<a name="line.757"></a>
+<span class="sourceLineNo">758</span> // Nonce Procedure helpers<a name="line.758"></a>
+<span class="sourceLineNo">759</span> // ==========================================================================<a name="line.759"></a>
+<span class="sourceLineNo">760</span> /**<a name="line.760"></a>
+<span class="sourceLineNo">761</span> * Create a NonceKey from the specified nonceGroup and nonce.<a name="line.761"></a>
+<span class="sourceLineNo">762</span> * @param nonceGroup the group to use for the {@link NonceKey}<a name="line.762"></a>
+<span class="sourceLineNo">763</span> * @param nonce the nonce to use in the {@link NonceKey}<a name="line.763"></a>
+<span class="sourceLineNo">764</span> * @return the generated NonceKey<a name="line.764"></a>
+<span class="sourceLineNo">765</span> */<a name="line.765"></a>
+<span class="sourceLineNo">766</span> public NonceKey createNonceKey(final long nonceGroup, final long nonce) {<a name="line.766"></a>
+<span class="sourceLineNo">767</span> return (nonce == HConstants.NO_NONCE) ? null : new NonceKey(nonceGroup, nonce);<a name="line.767"></a>
+<span class="sourceLineNo">768</span> }<a name="line.768"></a>
+<span class="sourceLineNo">769</span><a name="line.769"></a>
+<span class="sourceLineNo">770</span> /**<a name="line.770"></a>
+<span class="sourceLineNo">771</span> * Register a nonce for a procedure that is going to be submitted.<a name="line.771"></a>
+<span class="sourceLineNo">772</span> * A procId will be reserved and on submitProcedure(),<a name="line.772"></a>
+<span class="sourceLineNo">773</span> * the procedure with the specified nonce will take the reserved ProcId.<a name="line.773"></a>
+<span class="sourceLineNo">774</span> * If someone already reserved the nonce, this method will return the procId reserved,<a name="line.774"></a>
+<span class="sourceLineNo">775</span> * otherwise an invalid procId will be returned. and the caller should procede<a name="line.775"></a>
+<span class="sourceLineNo">776</span> * and submit the procedure.<a name="line.776"></a>
+<span class="sourceLineNo">777</span> *<a name="line.777"></a>
+<span class="sourceLineNo">778</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.778"></a>
+<span class="sourceLineNo">779</span> * @return the procId associated with the nonce, if any otherwise an invalid procId.<a name="line.779"></a>
+<span class="sourceLineNo">780</span> */<a name="line.780"></a>
+<span class="sourceLineNo">781</span> public long registerNonce(final NonceKey nonceKey) {<a name="line.781"></a>
+<span class="sourceLineNo">782</span> if (nonceKey == null) {<a name="line.782"></a>
+<span class="sourceLineNo">783</span> return -1;<a name="line.783"></a>
+<span class="sourceLineNo">784</span> }<a name="line.784"></a>
+<span class="sourceLineNo">785</span><a name="line.785"></a>
+<span class="sourceLineNo">786</span> // check if we have already a Reserved ID for the nonce<a name="line.786"></a>
+<span class="sourceLineNo">787</span> Long oldProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.787"></a>
+<span class="sourceLineNo">788</span> if (oldProcId == null) {<a name="line.788"></a>
+<span class="sourceLineNo">789</span> // reserve a new Procedure ID, this will be associated with the nonce<a name="line.789"></a>
+<span class="sourceLineNo">790</span> // and the procedure submitted with the specified nonce will use this ID.<a name="line.790"></a>
+<span class="sourceLineNo">791</span> final long newProcId = nextProcId();<a name="line.791"></a>
+<span class="sourceLineNo">792</span> oldProcId = nonceKeysToProcIdsMap.putIfAbsent(nonceKey, newProcId);<a name="line.792"></a>
+<span class="sourceLineNo">793</span> if (oldProcId == null) {<a name="line.793"></a>
+<span class="sourceLineNo">794</span> return -1;<a name="line.794"></a>
+<span class="sourceLineNo">795</span> }<a name="line.795"></a>
+<span class="sourceLineNo">796</span> }<a name="line.796"></a>
+<span class="sourceLineNo">797</span><a name="line.797"></a>
+<span class="sourceLineNo">798</span> // we found a registered nonce, but the procedure may not have been submitted yet.<a name="line.798"></a>
+<span class="sourceLineNo">799</span> // since the client expect the procedure to be submitted, spin here until it is.<a name="line.799"></a>
+<span class="sourceLineNo">800</span> final boolean traceEnabled = LOG.isTraceEnabled();<a name="line.800"></a>
+<span class="sourceLineNo">801</span> while (isRunning() &&<a name="line.801"></a>
+<span class="sourceLineNo">802</span> !(procedures.containsKey(oldProcId) || completed.containsKey(oldProcId)) &&<a name="line.802"></a>
+<span class="sourceLineNo">803</span> nonceKeysToProcIdsMap.containsKey(nonceKey)) {<a name="line.803"></a>
+<span class="sourceLineNo">804</span> if (traceEnabled) {<a name="line.804"></a>
+<span class="sourceLineNo">805</span> LOG.trace("Waiting for pid=" + oldProcId.longValue() + " to be submitted");<a name="line.805"></a>
+<span class="sourceLineNo">806</span> }<a name="line.806"></a>
+<span class="sourceLineNo">807</span> Threads.sleep(100);<a name="line.807"></a>
+<span class="sourceLineNo">808</span> }<a name="line.808"></a>
+<span class="sourceLineNo">809</span> return oldProcId.longValue();<a name="line.809"></a>
+<span class="sourceLineNo">810</span> }<a name="line.810"></a>
+<span class="sourceLineNo">811</span><a name="line.811"></a>
+<span class="sourceLineNo">812</span> /**<a name="line.812"></a>
+<span class="sourceLineNo">813</span> * Remove the NonceKey if the procedure was not submitted to the executor.<a name="line.813"></a>
+<span class="sourceLineNo">814</span> * @param nonceKey A unique identifier for this operation from the client or process.<a name="line.814"></a>
+<span class="sourceLineNo">815</span> */<a name="line.815"></a>
+<span class="sourceLineNo">816</span> public void unregisterNonceIfProcedureWasNotSubmitted(final NonceKey nonceKey) {<a name="line.816"></a>
+<span class="sourceLineNo">817</span> if (nonceKey == null) {<a name="line.817"></a>
+<span class="sourceLineNo">818</span> return;<a name="line.818"></a>
+<span class="sourceLineNo">819</span> }<a name="line.819"></a>
+<span class="sourceLineNo">820</span><a name="line.820"></a>
+<span class="sourceLineNo">821</span> final Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.821"></a>
+<span class="sourceLineNo">822</span> if (procId == null) {<a name="line.822"></a>
+<span class="sourceLineNo">823</span> return;<a name="line.823"></a>
+<span class="sourceLineNo">824</span> }<a name="line.824"></a>
+<span class="sourceLineNo">825</span><a name="line.825"></a>
+<span class="sourceLineNo">826</span> // if the procedure was not submitted, remove the nonce<a name="line.826"></a>
+<span class="sourceLineNo">827</span> if (!(procedures.containsKey(procId) || completed.containsKey(procId))) {<a name="line.827"></a>
+<span class="sourceLineNo">828</span> nonceKeysToProcIdsMap.remove(nonceKey);<a name="line.828"></a>
+<span class="sourceLineNo">829</span> }<a name="line.829"></a>
+<span class="sourceLineNo">830</span> }<a name="line.830"></a>
+<span class="sourceLineNo">831</span><a name="line.831"></a>
+<span class="sourceLineNo">832</span> /**<a name="line.832"></a>
+<span class="sourceLineNo">833</span> * If the failure failed before submitting it, we may want to give back the<a name="line.833"></a>
+<span class="sourceLineNo">834</span> * same error to the requests with the same nonceKey.<a name="line.834"></a>
+<span class="sourceLineNo">835</span> *<a name="line.835"></a>
+<span class="sourceLineNo">836</span> * @param nonceKey A unique identifier for this operation from the client or process<a name="line.836"></a>
+<span class="sourceLineNo">837</span> * @param procName name of the procedure, used to inform the user<a name="line.837"></a>
+<span class="sourceLineNo">838</span> * @param procOwner name of the owner of the procedure, used to inform the user<a name="line.838"></a>
+<span class="sourceLineNo">839</span> * @param exception the failure to report to the user<a name="line.839"></a>
+<span class="sourceLineNo">840</span> */<a name="line.840"></a>
+<span class="sourceLineNo">841</span> public void setFailureResultForNonce(NonceKey nonceKey, String procName, User procOwner,<a name="line.841"></a>
+<span class="sourceLineNo">842</span> IOException exception) {<a name="line.842"></a>
+<span class="sourceLineNo">843</span> if (nonceKey == null) {<a name="line.843"></a>
+<span class="sourceLineNo">844</span> return;<a name="line.844"></a>
+<span class="sourceLineNo">845</span> }<a name="line.845"></a>
+<span class="sourceLineNo">846</span><a name="line.846"></a>
+<span class="sourceLineNo">847</span> Long procId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.847"></a>
+<span class="sourceLineNo">848</span> if (procId == null || completed.containsKey(procId)) {<a name="line.848"></a>
+<span class="sourceLineNo">849</span> return;<a name="line.849"></a>
+<span class="sourceLineNo">850</span> }<a name="line.850"></a>
+<span class="sourceLineNo">851</span><a name="line.851"></a>
+<span class="sourceLineNo">852</span> completed.computeIfAbsent(procId, (key) -> {<a name="line.852"></a>
+<span class="sourceLineNo">853</span> Procedure<TEnvironment> proc = new FailedProcedure<>(procId.longValue(),<a name="line.853"></a>
+<span class="sourceLineNo">854</span> procName, procOwner, nonceKey, exception);<a name="line.854"></a>
+<span class="sourceLineNo">855</span><a name="line.855"></a>
+<span class="sourceLineNo">856</span> return new CompletedProcedureRetainer<>(proc);<a name="line.856"></a>
+<span class="sourceLineNo">857</span> });<a name="line.857"></a>
+<span class="sourceLineNo">858</span> }<a name="line.858"></a>
+<span class="sourceLineNo">859</span><a name="line.859"></a>
+<span class="sourceLineNo">860</span> // ==========================================================================<a name="line.860"></a>
+<span class="sourceLineNo">861</span> // Submit/Abort Procedure<a name="line.861"></a>
+<span class="sourceLineNo">862</span> // ==========================================================================<a name="line.862"></a>
+<span class="sourceLineNo">863</span> /**<a name="line.863"></a>
+<span class="sourceLineNo">864</span> * Add a new root-procedure to the executor.<a name="line.864"></a>
+<span class="sourceLineNo">865</span> * @param proc the new procedure to execute.<a name="line.865"></a>
+<span class="sourceLineNo">866</span> * @return the procedure id, that can be used to monitor the operation<a name="line.866"></a>
+<span class="sourceLineNo">867</span> */<a name="line.867"></a>
+<span class="sourceLineNo">868</span> public long submitProcedure(Procedure<TEnvironment> proc) {<a name="line.868"></a>
+<span class="sourceLineNo">869</span> return submitProcedure(proc, null);<a name="line.869"></a>
+<span class="sourceLineNo">870</span> }<a name="line.870"></a>
+<span class="sourceLineNo">871</span><a name="line.871"></a>
+<span class="sourceLineNo">872</span> /**<a name="line.872"></a>
+<span class="sourceLineNo">873</span> * Bypass a procedure. If the procedure is set to bypass, all the logic in<a name="line.873"></a>
+<span class="sourceLineNo">874</span> * execute/rollback will be ignored and it will return success, whatever.<a name="line.874"></a>
+<span class="sourceLineNo">875</span> * It is used to recover buggy stuck procedures, releasing the lock resources<a name="line.875"></a>
+<span class="sourceLineNo">876</span> * and letting other procedures run. Bypassing one procedure (and its ancestors will<a name="line.876"></a>
+<span class="sourceLineNo">877</span> * be bypassed automatically) may leave the cluster in a middle state, e.g. region<a name="line.877"></a>
+<span class="sourceLineNo">878</span> * not assigned, or some hdfs files left behind. After getting rid of those stuck procedures,<a name="line.878"></a>
+<span class="sourceLineNo">879</span> * the operators may have to do some clean up on hdfs or schedule some assign procedures<a name="line.879"></a>
+<span class="sourceLineNo">880</span> * to let region online. DO AT YOUR OWN RISK.<a name="line.880"></a>
<span class="sourceLineNo">881</span> * <p><a name="line.881"></a>
-<span class="sourceLineNo">882</span> * If all the requirements are meet, the procedure and its ancestors will be<a name="line.882"></a>
-<span class="sourceLineNo">883</span> * bypassed and persisted to WAL.<a name="line.883"></a>
-<span class="sourceLineNo">884</span> *<a name="line.884"></a>
-<span class="sourceLineNo">885</span> * <p><a name="line.885"></a>
-<span class="sourceLineNo">886</span> * If the procedure is in WAITING state, will set it to RUNNABLE add it to run queue.<a name="line.886"></a>
-<span class="sourceLineNo">887</span> * TODO: What about WAITING_TIMEOUT?<a name="line.887"></a>
-<span class="sourceLineNo">888</span> * @param pids the procedure id<a name="line.888"></a>
-<span class="sourceLineNo">889</span> * @param lockWait time to wait lock<a name="line.889"></a>
-<span class="sourceLineNo">890</span> * @param force if force set to true, we will bypass the procedure even if it is executing.<a name="line.890"></a>
-<span class="sourceLineNo">891</span> * This is for procedures which can't break out during executing(due to bug, mostly)<a name="line.891"></a>
-<span class="sourceLineNo">892</span> * In this case, bypassing the procedure is not enough, since it is already stuck<a name="line.892"></a>
-<span class="sourceLineNo">893</span> * there. We need to restart the master after bypassing, and letting the problematic<a name="line.893"></a>
-<span class="sourceLineNo">894</span> * procedure to execute wth bypass=true, so in that condition, the procedure can be<a name="line.894"></a>
-<span class="sourceLineNo">895</span> * successfully bypassed.<a name="line.895"></a>
-<span class="sourceLineNo">896</span> * @param recursive We will do an expensive search for children of each pid. EXPENSIVE!<a name="line.896"></a>
-<span class="sourceLineNo">897</span> * @return true if bypass success<a name="line.897"></a>
-<span class="sourceLineNo">898</span> * @throws IOException IOException<a name="line.898"></a>
-<span class="sourceLineNo">899</span> */<a name="line.899"></a>
-<span class="sourceLineNo">900</span> public List<Boolean> bypassProcedure(List<Long> pids, long lockWait, boolean force,<a name="line.900"></a>
-<span class="sourceLineNo">901</span> boolean recursive)<a name="line.901"></a>
-<span class="sourceLineNo">902</span> throws IOException {<a name="line.902"></a>
-<span class="sourceLineNo">903</span> List<Boolean> result = new ArrayList<Boolean>(pids.size());<a name="line.903"></a>
-<span class="sourceLineNo">904</span> for(long pid: pids) {<a name="line.904"></a>
-<span class="sourceLineNo">905</span> result.add(bypassProcedure(pid, lockWait, force, recursive));<a name="line.905"></a>
-<span class="sourceLineNo">906</span> }<a name="line.906"></a>
-<span class="sourceLineNo">907</span> return result;<a name="line.907"></a>
-<span class="sourceLineNo">908</span> }<a name="line.908"></a>
-<span class="sourceLineNo">909</span><a name="line.909"></a>
-<span class="sourceLineNo">910</span> boolean bypassProcedure(long pid, long lockWait, boolean override, boolean recursive)<a name="line.910"></a>
-<span class="sourceLineNo">911</span> throws IOException {<a name="line.911"></a>
-<span class="sourceLineNo">912</span> Preconditions.checkArgument(lockWait > 0, "lockWait should be positive");<a name="line.912"></a>
-<span class="sourceLineNo">913</span> final Procedure<TEnvironment> procedure = getProcedure(pid);<a name="line.913"></a>
-<span class="sourceLineNo">914</span> if (procedure == null) {<a name="line.914"></a>
-<span class="sourceLineNo">915</span> LOG.debug("Procedure pid={} does not exist, skipping bypass", pid);<a name="line.915"></a>
-<span class="sourceLineNo">916</span> return false;<a name="line.916"></a>
-<span class="sourceLineNo">917</span> }<a name="line.917"></a>
-<span class="sourceLineNo">918</span><a name="line.918"></a>
-<span class="sourceLineNo">919</span> LOG.debug("Begin bypass {} with lockWait={}, override={}, recursive={}",<a name="line.919"></a>
-<span class="sourceLineNo">920</span> procedure, lockWait, override, recursive);<a name="line.920"></a>
-<span class="sourceLineNo">921</span> IdLock.Entry lockEntry = procExecutionLock.tryLockEntry(procedure.getProcId(), lockWait);<a name="line.921"></a>
-<span class="sourceLineNo">922</span> if (lockEntry == null && !override) {<a name="line.922"></a>
-<span class="sourceLineNo">923</span> LOG.debug("Waited {} ms, but {} is still running, skipping bypass with force={}",<a name="line.923"></a>
-<span class="sourceLineNo">924</span> lockWait, procedure, override);<a name="line.924"></a>
-<span class="sourceLineNo">925</span> return false;<a name="line.925"></a>
-<span class="sourceLineNo">926</span> } else if (lockEntry == null) {<a name="line.926"></a>
-<span class="sourceLineNo">927</span> LOG.debug("Waited {} ms, but {} is still running, begin bypass with force={}",<a name="line.927"></a>
-<span class="sourceLineNo">928</span> lockWait, procedure, override);<a name="line.928"></a>
-<span class="sourceLineNo">929</span> }<a name="line.929"></a>
-<span class="sourceLineNo">930</span> try {<a name="line.930"></a>
-<span class="sourceLineNo">931</span> // check whether the procedure is already finished<a name="line.931"></a>
-<span class="sourceLineNo">932</span> if (procedure.isFinished()) {<a name="line.932"></a>
-<span class="sourceLineNo">933</span> LOG.debug("{} is already finished, skipping bypass", procedure);<a name="line.933"></a>
-<span class="sourceLineNo">934</span> return false;<a name="line.934"></a>
-<span class="sourceLineNo">935</span> }<a name="line.935"></a>
-<span class="sourceLineNo">936</span><a name="line.936"></a>
-<span class="sourceLineNo">937</span> if (procedure.hasChildren()) {<a name="line.937"></a>
-<span class="sourceLineNo">938</span> if (recursive) {<a name="line.938"></a>
-<span class="sourceLineNo">939</span> // EXPENSIVE. Checks each live procedure of which there could be many!!!<a name="line.939"></a>
-<span class="sourceLineNo">940</span> // Is there another way to get children of a procedure?<a name="line.940"></a>
-<span class="sourceLineNo">941</span> LOG.info("Recursive bypass on children of pid={}", procedure.getProcId());<a name="line.941"></a>
-<span class="sourceLineNo">942</span> this.procedures.forEachValue(1 /*Single-threaded*/,<a name="line.942"></a>
-<span class="sourceLineNo">943</span> // Transformer<a name="line.943"></a>
-<span class="sourceLineNo">944</span> v -> v.getParentProcId() == procedure.getProcId()? v: null,<a name="line.944"></a>
-<span class="sourceLineNo">945</span> // Consumer<a name="line.945"></a>
-<span class="sourceLineNo">946</span> v -> {<a name="line.946"></a>
-<span class="sourceLineNo">947</span> try {<a name="line.947"></a>
-<span class="sourceLineNo">948</span> bypassProcedure(v.getProcId(), lockWait, override, recursive);<a name="line.948"></a>
-<span class="sourceLineNo">949</span> } catch (IOException e) {<a name="line.949"></a>
-<span class="sourceLineNo">950</span> LOG.warn("Recursive bypass of pid={}", v.getProcId(), e);<a name="line.950"></a>
-<span class="sourceLineNo">951</span> }<a name="line.951"></a>
-<span class="sourceLineNo">952</span> });<a name="line.952"></a>
-<span class="sourceLineNo">953</span> } else {<a name="line.953"></a>
-<span class="sourceLineNo">954</span> LOG.debug("{} has children, skipping bypass", procedure);<a name="line.954"></a>
-<span class="sourceLineNo">955</span> return false;<a name="line.955"></a>
-<span class="sourceLineNo">956</span> }<a name="line.956"></a>
-<span class="sourceLineNo">957</span> }<a name="line.957"></a>
-<span class="sourceLineNo">958</span><a name="line.958"></a>
-<span class="sourceLineNo">959</span> // If the procedure has no parent or no child, we are safe to bypass it in whatever state<a name="line.959"></a>
-<span class="sourceLineNo">960</span> if (procedure.hasParent() && procedure.getState() != ProcedureState.RUNNABLE<a name="line.960"></a>
-<span class="sourceLineNo">961</span> && procedure.getState() != ProcedureState.WAITING<a name="line.961"></a>
-<span class="sourceLineNo">962</span> && procedure.getState() != ProcedureState.WAITING_TIMEOUT) {<a name="line.962"></a>
-<span class="sourceLineNo">963</span> LOG.debug("Bypassing procedures in RUNNABLE, WAITING and WAITING_TIMEOUT states "<a name="line.963"></a>
-<span class="sourceLineNo">964</span> + "(with no parent), {}",<a name="line.964"></a>
-<span class="sourceLineNo">965</span> procedure);<a name="line.965"></a>
-<span class="sourceLineNo">966</span> // Question: how is the bypass done here?<a name="line.966"></a>
-<span class="sourceLineNo">967</span> return false;<a name="line.967"></a>
-<span class="sourceLineNo">968</span> }<a name="line.968"></a>
-<span class="sourceLineNo">969</span><a name="line.969"></a>
-<span class="sourceLineNo">970</span> // Now, the procedure is not finished, and no one can execute it since we take the lock now<a name="line.970"></a>
-<span class="sourceLineNo">971</span> // And we can be sure that its ancestor is not running too, since their child has not<a name="line.971"></a>
-<span class="sourceLineNo">972</span> // finished yet<a name="line.972"></a>
-<span class="sourceLineNo">973</span> Procedure<TEnvironment> current = procedure;<a name="line.973"></a>
-<span class="sourceLineNo">974</span> while (current != null) {<a name="line.974"></a>
-<span class="sourceLineNo">975</span> LOG.debug("Bypassing {}", current);<a name="line.975"></a>
-<span class="sourceLineNo">976</span> current.bypass(getEnvironment());<a name="line.976"></a>
-<span class="sourceLineNo">977</span> store.update(procedure);<a name="line.977"></a>
-<span class="sourceLineNo">978</span> long parentID = current.getParentProcId();<a name="line.978"></a>
-<span class="sourceLineNo">979</span> current = getProcedure(parentID);<a name="line.979"></a>
-<span class="sourceLineNo">980</span> }<a name="line.980"></a>
-<span class="sourceLineNo">981</span><a name="line.981"></a>
-<span class="sourceLineNo">982</span> //wake up waiting procedure, already checked there is no child<a name="line.982"></a>
-<span class="sourceLineNo">983</span> if (procedure.getState() == ProcedureState.WAITING) {<a name="line.983"></a>
-<span class="sourceLineNo">984</span> procedure.setState(ProcedureState.RUNNABLE);<a name="line.984"></a>
-<span class="sourceLineNo">985</span> store.update(procedure);<a name="line.985"></a>
-<span class="sourceLineNo">986</span> }<a name="line.986"></a>
-<span class="sourceLineNo">987</span><a name="line.987"></a>
-<span class="sourceLineNo">988</span> // If state of procedure is WAITING_TIMEOUT, we can directly submit it to the scheduler.<a name="line.988"></a>
-<span class="sourceLineNo">989</span> // Instead we should remove it from timeout Executor queue and tranfer its state to RUNNABLE<a name="line.989"></a>
-<span class="sourceLineNo">990</span> if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.990"></a>
-<span class="sourceLineNo">991</span> LOG.debug("transform procedure {} from WAITING_TIMEOUT to RUNNABLE", procedure);<a name="line.991"></a>
-<span class="sourceLineNo">992</span> if (timeoutExecutor.remove(procedure)) {<a name="line.992"></a>
-<span class="sourceLineNo">993</span> LOG.debug("removed procedure {} from timeoutExecutor", procedure);<a name="line.993"></a>
-<span class="sourceLineNo">994</span> timeoutExecutor.executeTimedoutProcedure(procedure);<a name="line.994"></a>
-<span class="sourceLineNo">995</span> }<a name="line.995"></a>
-<span class="sourceLineNo">996</span> } else if (lockEntry != null) {<a name="line.996"></a>
-<span class="sourceLineNo">997</span> scheduler.addFront(procedure);<a name="line.997"></a>
-<span class="sourceLineNo">998</span> LOG.debug("Bypassing {} and its ancestors successfully, adding to queue", procedure);<a name="line.998"></a>
-<span class="sourceLineNo">999</span> } else {<a name="line.999"></a>
-<span class="sourceLineNo">1000</span> // If we don't have the lock, we can't re-submit the queue,<a name="line.1000"></a>
-<span class="sourceLineNo">1001</span> // since it is already executing. To get rid of the stuck situation, we<a name="line.1001"></a>
-<span class="sourceLineNo">1002</span> // need to restart the master. With the procedure set to bypass, the procedureExecutor<a name="line.1002"></a>
-<span class="sourceLineNo">1003</span> // will bypass it and won't get stuck again.<a name="line.1003"></a>
-<span class="sourceLineNo">1004</span> LOG.debug("Bypassing {} and its ancestors successfully, but since it is already running, "<a name="line.1004"></a>
-<span class="sourceLineNo">1005</span> + "skipping add to queue",<a name="line.1005"></a>
-<span class="sourceLineNo">1006</span> procedure);<a name="line.1006"></a>
-<span class="sourceLineNo">1007</span> }<a name="line.1007"></a>
-<span class="sourceLineNo">1008</span> return true;<a name="line.1008"></a>
-<span class="sourceLineNo">1009</span><a name="line.1009"></a>
-<span class="sourceLineNo">1010</span> } finally {<a name="line.1010"></a>
-<span class="sourceLineNo">1011</span> if (lockEntry != null) {<a name="line.1011"></a>
-<span class="sourceLineNo">1012</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1012"></a>
-<span class="sourceLineNo">1013</span> }<a name="line.1013"></a>
-<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
-<span class="sourceLineNo">1015</span> }<a name="line.1015"></a>
+<span class="sourceLineNo">882</span> * A procedure can be bypassed only if<a name="line.882"></a>
+<span class="sourceLineNo">883</span> * 1. The procedure is in state of RUNNABLE, WAITING, WAITING_TIMEOUT<a name="line.883"></a>
+<span class="sourceLineNo">884</span> * or it is a root procedure without any child.<a name="line.884"></a>
+<span class="sourceLineNo">885</span> * 2. No other worker thread is executing it<a name="line.885"></a>
+<span class="sourceLineNo">886</span> * 3. No child procedure has been submitted<a name="line.886"></a>
+<span class="sourceLineNo">887</span> *<a name="line.887"></a>
+<span class="sourceLineNo">888</span> * <p><a name="line.888"></a>
+<span class="sourceLineNo">889</span> * If all the requirements are meet, the procedure and its ancestors will be<a name="line.889"></a>
+<span class="sourceLineNo">890</span> * bypassed and persisted to WAL.<a name="line.890"></a>
+<span class="sourceLineNo">891</span> *<a name="line.891"></a>
+<span class="sourceLineNo">892</span> * <p><a name="line.892"></a>
+<span class="sourceLineNo">893</span> * If the procedure is in WAITING state, will set it to RUNNABLE add it to run queue.<a name="line.893"></a>
+<span class="sourceLineNo">894</span> * TODO: What about WAITING_TIMEOUT?<a name="line.894"></a>
+<span class="sourceLineNo">895</span> * @param pids the procedure id<a name="line.895"></a>
+<span class="sourceLineNo">896</span> * @param lockWait time to wait lock<a name="line.896"></a>
+<span class="sourceLineNo">897</span> * @param force if force set to true, we will bypass the procedure even if it is executing.<a name="line.897"></a>
+<span class="sourceLineNo">898</span> * This is for procedures which can't break out during executing(due to bug, mostly)<a name="line.898"></a>
+<span class="sourceLineNo">899</span> * In this case, bypassing the procedure is not enough, since it is already stuck<a name="line.899"></a>
+<span class="sourceLineNo">900</span> * there. We need to restart the master after bypassing, and letting the problematic<a name="line.900"></a>
+<span class="sourceLineNo">901</span> * procedure to execute wth bypass=true, so in that condition, the procedure can be<a name="line.901"></a>
+<span class="sourceLineNo">902</span> * successfully bypassed.<a name="line.902"></a>
+<span class="sourceLineNo">903</span> * @param recursive We will do an expensive search for children of each pid. EXPENSIVE!<a name="line.903"></a>
+<span class="sourceLineNo">904</span> * @return true if bypass success<a name="line.904"></a>
+<span class="sourceLineNo">905</span> * @throws IOException IOException<a name="line.905"></a>
+<span class="sourceLineNo">906</span> */<a name="line.906"></a>
+<span class="sourceLineNo">907</span> public List<Boolean> bypassProcedure(List<Long> pids, long lockWait, boolean force,<a name="line.907"></a>
+<span class="sourceLineNo">908</span> boolean recursive)<a name="line.908"></a>
+<span class="sourceLineNo">909</span> throws IOException {<a name="line.909"></a>
+<span class="sourceLineNo">910</span> List<Boolean> result = new ArrayList<Boolean>(pids.size());<a name="line.910"></a>
+<span class="sourceLineNo">911</span> for(long pid: pids) {<a name="line.911"></a>
+<span class="sourceLineNo">912</span> result.add(bypassProcedure(pid, lockWait, force, recursive));<a name="line.912"></a>
+<span class="sourceLineNo">913</span> }<a name="line.913"></a>
+<span class="sourceLineNo">914</span> return result;<a name="line.914"></a>
+<span class="sourceLineNo">915</span> }<a name="line.915"></a>
+<span class="sourceLineNo">916</span><a name="line.916"></a>
+<span class="sourceLineNo">917</span> boolean bypassProcedure(long pid, long lockWait, boolean override, boolean recursive)<a name="line.917"></a>
+<span class="sourceLineNo">918</span> throws IOException {<a name="line.918"></a>
+<span class="sourceLineNo">919</span> Preconditions.checkArgument(lockWait > 0, "lockWait should be positive");<a name="line.919"></a>
+<span class="sourceLineNo">920</span> final Procedure<TEnvironment> procedure = getProcedure(pid);<a name="line.920"></a>
+<span class="sourceLineNo">921</span> if (procedure == null) {<a name="line.921"></a>
+<span class="sourceLineNo">922</span> LOG.debug("Procedure pid={} does not exist, skipping bypass", pid);<a name="line.922"></a>
+<span class="sourceLineNo">923</span> return false;<a name="line.923"></a>
+<span class="sourceLineNo">924</span> }<a name="line.924"></a>
+<span class="sourceLineNo">925</span><a name="line.925"></a>
+<span class="sourceLineNo">926</span> LOG.debug("Begin bypass {} with lockWait={}, override={}, recursive={}",<a name="line.926"></a>
+<span class="sourceLineNo">927</span> procedure, lockWait, override, recursive);<a name="line.927"></a>
+<span class="sourceLineNo">928</span> IdLock.Entry lockEntry = procExecutionLock.tryLockEntry(procedure.getProcId(), lockWait);<a name="line.928"></a>
+<span class="sourceLineNo">929</span> if (lockEntry == null && !override) {<a name="line.929"></a>
+<span class="sourceLineNo">930</span> LOG.debug("Waited {} ms, but {} is still running, skipping bypass with force={}",<a name="line.930"></a>
+<span class="sourceLineNo">931</span> lockWait, procedure, override);<a name="line.931"></a>
+<span class="sourceLineNo">932</span> return false;<a name="line.932"></a>
+<span class="sourceLineNo">933</span> } else if (lockEntry == null) {<a name="line.933"></a>
+<span class="sourceLineNo">934</span> LOG.debug("Waited {} ms, but {} is still running, begin bypass with force={}",<a name="line.934"></a>
+<span class="sourceLineNo">935</span> lockWait, procedure, override);<a name="line.935"></a>
+<span class="sourceLineNo">936</span> }<a name="line.936"></a>
+<span class="sourceLineNo">937</span> try {<a name="line.937"></a>
+<span class="sourceLineNo">938</span> // check whether the procedure is already finished<a name="line.938"></a>
+<span class="sourceLineNo">939</span> if (procedure.isFinished()) {<a name="line.939"></a>
+<span class="sourceLineNo">940</span> LOG.debug("{} is already finished, skipping bypass", procedure);<a name="line.940"></a>
+<span class="sourceLineNo">941</span> return false;<a name="line.941"></a>
+<span class="sourceLineNo">942</span> }<a name="line.942"></a>
+<span class="sourceLineNo">943</span><a name="line.943"></a>
+<span class="sourceLineNo">944</span> if (procedure.hasChildren()) {<a name="line.944"></a>
+<span class="sourceLineNo">945</span> if (recursive) {<a name="line.945"></a>
+<span class="sourceLineNo">946</span> // EXPENSIVE. Checks each live procedure of which there could be many!!!<a name="line.946"></a>
+<span class="sourceLineNo">947</span> // Is there another way to get children of a procedure?<a name="line.947"></a>
+<span class="sourceLineNo">948</span> LOG.info("Recursive bypass on children of pid={}", procedure.getProcId());<a name="line.948"></a>
+<span class="sourceLineNo">949</span> this.procedures.forEachValue(1 /*Single-threaded*/,<a name="line.949"></a>
+<span class="sourceLineNo">950</span> // Transformer<a name="line.950"></a>
+<span class="sourceLineNo">951</span> v -> v.getParentProcId() == procedure.getProcId()? v: null,<a name="line.951"></a>
+<span class="sourceLineNo">952</span> // Consumer<a name="line.952"></a>
+<span class="sourceLineNo">953</span> v -> {<a name="line.953"></a>
+<span class="sourceLineNo">954</span> try {<a name="line.954"></a>
+<span class="sourceLineNo">955</span> bypassProcedure(v.getProcId(), lockWait, override, recursive);<a name="line.955"></a>
+<span class="sourceLineNo">956</span> } catch (IOException e) {<a name="line.956"></a>
+<span class="sourceLineNo">957</span> LOG.warn("Recursive bypass of pid={}", v.getProcId(), e);<a name="line.957"></a>
+<span class="sourceLineNo">958</span> }<a name="line.958"></a>
+<span class="sourceLineNo">959</span> });<a name="line.959"></a>
+<span class="sourceLineNo">960</span> } else {<a name="line.960"></a>
+<span class="sourceLineNo">961</span> LOG.debug("{} has children, skipping bypass", procedure);<a name="line.961"></a>
+<span class="sourceLineNo">962</span> return false;<a name="line.962"></a>
+<span class="sourceLineNo">963</span> }<a name="line.963"></a>
+<span class="sourceLineNo">964</span> }<a name="line.964"></a>
+<span class="sourceLineNo">965</span><a name="line.965"></a>
+<span class="sourceLineNo">966</span> // If the procedure has no parent or no child, we are safe to bypass it in whatever state<a name="line.966"></a>
+<span class="sourceLineNo">967</span> if (procedure.hasParent() && procedure.getState() != ProcedureState.RUNNABLE<a name="line.967"></a>
+<span class="sourceLineNo">968</span> && procedure.getState() != ProcedureState.WAITING<a name="line.968"></a>
+<span class="sourceLineNo">969</span> && procedure.getState() != ProcedureState.WAITING_TIMEOUT) {<a name="line.969"></a>
+<span class="sourceLineNo">970</span> LOG.debug("Bypassing procedures in RUNNABLE, WAITING and WAITING_TIMEOUT states "<a name="line.970"></a>
+<span class="sourceLineNo">971</span> + "(with no parent), {}",<a name="line.971"></a>
+<span class="sourceLineNo">972</span> procedure);<a name="line.972"></a>
+<span class="sourceLineNo">973</span> // Question: how is the bypass done here?<a name="line.973"></a>
+<span class="sourceLineNo">974</span> return false;<a name="line.974"></a>
+<span class="sourceLineNo">975</span> }<a name="line.975"></a>
+<span class="sourceLineNo">976</span><a name="line.976"></a>
+<span class="sourceLineNo">977</span> // Now, the procedure is not finished, and no one can execute it since we take the lock now<a name="line.977"></a>
+<span class="sourceLineNo">978</span> // And we can be sure that its ancestor is not running too, since their child has not<a name="line.978"></a>
+<span class="sourceLineNo">979</span> // finished yet<a name="line.979"></a>
+<span class="sourceLineNo">980</span> Procedure<TEnvironment> current = procedure;<a name="line.980"></a>
+<span class="sourceLineNo">981</span> while (current != null) {<a name="line.981"></a>
+<span class="sourceLineNo">982</span> LOG.debug("Bypassing {}", current);<a name="line.982"></a>
+<span class="sourceLineNo">983</span> current.bypass(getEnvironment());<a name="line.983"></a>
+<span class="sourceLineNo">984</span> store.update(procedure);<a name="line.984"></a>
+<span class="sourceLineNo">985</span> long parentID = current.getParentProcId();<a name="line.985"></a>
+<span class="sourceLineNo">986</span> current = getProcedure(parentID);<a name="line.986"></a>
+<span class="sourceLineNo">987</span> }<a name="line.987"></a>
+<span class="sourceLineNo">988</span><a name="line.988"></a>
+<span class="sourceLineNo">989</span> //wake up waiting procedure, already checked there is no child<a name="line.989"></a>
+<span class="sourceLineNo">990</span> if (procedure.getState() == ProcedureState.WAITING) {<a name="line.990"></a>
+<span class="sourceLineNo">991</span> procedure.setState(ProcedureState.RUNNABLE);<a name="line.991"></a>
+<span class="sourceLineNo">992</span> store.update(procedure);<a name="line.992"></a>
+<span class="sourceLineNo">993</span> }<a name="line.993"></a>
+<span class="sourceLineNo">994</span><a name="line.994"></a>
+<span class="sourceLineNo">995</span> // If state of procedure is WAITING_TIMEOUT, we can directly submit it to the scheduler.<a name="line.995"></a>
+<span class="sourceLineNo">996</span> // Instead we should remove it from timeout Executor queue and tranfer its state to RUNNABLE<a name="line.996"></a>
+<span class="sourceLineNo">997</span> if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.997"></a>
+<span class="sourceLineNo">998</span> LOG.debug("transform procedure {} from WAITING_TIMEOUT to RUNNABLE", procedure);<a name="line.998"></a>
+<span class="sourceLineNo">999</span> if (timeoutExecutor.remove(procedure)) {<a name="line.999"></a>
+<span class="sourceLineNo">1000</span> LOG.debug("removed procedure {} from timeoutExecutor", procedure);<a name="line.1000"></a>
+<span class="sourceLineNo">1001</span> timeoutExecutor.executeTimedoutProcedure(procedure);<a name="line.1001"></a>
+<span class="sourceLineNo">1002</span> }<a name="line.1002"></a>
+<span class="sourceLineNo">1003</span> } else if (lockEntry != null) {<a name="line.1003"></a>
+<span class="sourceLineNo">1004</span> scheduler.addFront(procedure);<a name="line.1004"></a>
+<span class="sourceLineNo">1005</span> LOG.debug("Bypassing {} and its ancestors successfully, adding to queue", procedure);<a name="line.1005"></a>
+<span class="sourceLineNo">1006</span> } else {<a name="line.1006"></a>
+<span class="sourceLineNo">1007</span> // If we don't have the lock, we can't re-submit the queue,<a name="line.1007"></a>
+<span class="sourceLineNo">1008</span> // since it is already executing. To get rid of the stuck situation, we<a name="line.1008"></a>
+<span class="sourceLineNo">1009</span> // need to restart the master. With the procedure set to bypass, the procedureExecutor<a name="line.1009"></a>
+<span class="sourceLineNo">1010</span> // will bypass it and won't get stuck again.<a name="line.1010"></a>
+<span class="sourceLineNo">1011</span> LOG.debug("Bypassing {} and its ancestors successfully, but since it is already running, "<a name="line.1011"></a>
+<span class="sourceLineNo">1012</span> + "skipping add to queue",<a name="line.1012"></a>
+<span class="sourceLineNo">1013</span> procedure);<a name="line.1013"></a>
+<span class="sourceLineNo">1014</span> }<a name="line.1014"></a>
+<span class="sourceLineNo">1015</span> return true;<a name="line.1015"></a>
<span class="sourceLineNo">1016</span><a name="line.1016"></a>
-<span class="sourceLineNo">1017</span> /**<a name="line.1017"></a>
-<span class="sourceLineNo">1018</span> * Add a new root-procedure to the executor.<a name="line.1018"></a>
-<span class="sourceLineNo">1019</span> * @param proc the new procedure to execute.<a name="line.1019"></a>
-<span class="sourceLineNo">1020</span> * @param nonceKey the registered unique identifier for this operation from the client or process.<a name="line.1020"></a>
-<span class="sourceLineNo">1021</span> * @return the procedure id, that can be used to monitor the operation<a name="line.1021"></a>
-<span class="sourceLineNo">1022</span> */<a name="line.1022"></a>
-<span class="sourceLineNo">1023</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",<a name="line.1023"></a>
-<span class="sourceLineNo">1024</span> justification = "FindBugs is blind to the check-for-null")<a name="line.1024"></a>
-<span class="sourceLineNo">1025</span> public long submitProcedure(Procedure<TEnvironment> proc, NonceKey nonceKey) {<a name="line.1025"></a>
-<span class="sourceLineNo">1026</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1026"></a>
-<span class="sourceLineNo">1027</span><a name="line.1027"></a>
-<span class="sourceLineNo">1028</span> prepareProcedure(proc);<a name="line.1028"></a>
-<span class="sourceLineNo">1029</span><a name="line.1029"></a>
-<span class="sourceLineNo">1030</span> final Long currentProcId;<a name="line.1030"></a>
-<span class="sourceLineNo">1031</span> if (nonceKey != null) {<a name="line.1031"></a>
-<span class="sourceLineNo">1032</span> currentProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.1032"></a>
-<span class="sourceLineNo">1033</span> Preconditions.checkArgument(currentProcId != null,<a name="line.1033"></a>
-<span class="sourceLineNo">1034</span> "Expected nonceKey=" + nonceKey + " to be reserved, use registerNonce(); proc=" + proc);<a name="line.1034"></a>
-<span class="sourceLineNo">1035</span> } else {<a name="line.1035"></a>
-<span class="sourceLineNo">1036</span> currentProcId = nextProcId();<a name="line.1036"></a>
-<span class="sourceLineNo">1037</span> }<a name="line.1037"></a>
-<span class="sourceLineNo">1038</span><a name="line.1038"></a>
-<span class="sourceLineNo">1039</span> // Initialize the procedure<a name="line.1039"></a>
-<span class="sourceLineNo">1040</span> proc.setNonceKey(nonceKey);<a name="line.1040"></a>
-<span class="sourceLineNo">1041</span> proc.setProcId(currentProcId.longValue());<a name="line.1041"></a>
-<span class="sourceLineNo">1042</span><a name="line.1042"></a>
-<span class="sourceLineNo">1043</span> // Commit the transaction<a name="line.1043"></a>
-<span class="sourceLineNo">1044</span> store.insert(proc, null);<a name="line.1044"></a>
-<span class="sourceLineNo">1045</span> LOG.debug("Stored {}", proc);<a name="line.1045"></a>
-<span class="sourceLineNo">1046</span><a name="line.1046"></a>
-<span class="sourceLineNo">1047</span> // Add the procedure to the executor<a name="line.1047"></a>
-<span class="sourceLineNo">1048</span> return pushProcedure(proc);<a name="line.1048"></a>
-<span class="sourceLineNo">1049</span> }<a name="line.1049"></a>
-<span class="sourceLineNo">1050</span><a name="line.1050"></a>
-<span class="sourceLineNo">1051</span> /**<a name="line.1051"></a>
-<span class="sourceLineNo">1052</span> * Add a set of new root-procedure to the executor.<a name="line.1052"></a>
-<span class="sourceLineNo">1053</span> * @param procs the new procedures to execute.<a name="line.1053"></a>
-<span class="sourceLineNo">1054</span> */<a name="line.1054"></a>
-<span class="sourceLineNo">1055</span> // TODO: Do we need to take nonces here?<a name="line.1055"></a>
-<span class="sourceLineNo">1056</span> public void submitProcedures(Procedure<TEnvironment>[] procs) {<a name="line.1056"></a>
-<span class="sourceLineNo">1057</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1057"></a>
-<span class="sourceLineNo">1058</span> if (procs == null || procs.length <= 0) {<a name="line.1058"></a>
-<span class="sourceLineNo">1059</span> return;<a name="line.1059"></a>
-<span class="sourceLineNo">1060</span> }<a name="line.1060"></a>
-<span class="sourceLineNo">1061</span><a name="line.1061"></a>
-<span class="sourceLineNo">1062</span> // Prepare procedure<a name="line.1062"></a>
-<span class="sourceLineNo">1063</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1063"></a>
-<span class="sourceLineNo">1064</span> prepareProcedure(procs[i]).setProcId(nextProcId());<a name="line.1064"></a>
-<span class="sourceLineNo">1065</span> }<a name="line.1065"></a>
-<span class="sourceLineNo">1066</span><a name="line.1066"></a>
-<span class="sourceLineNo">1067</span> // Commit the transaction<a name="line.1067"></a>
-<span class="sourceLineNo">1068</span> store.insert(procs);<a name="line.1068"></a>
-<span class="sourceLineNo">1069</span> if (LOG.isDebugEnabled()) {<a name="line.1069"></a>
-<span class="sourceLineNo">1070</span> LOG.debug("Stored " + Arrays.toString(procs));<a name="line.1070"></a>
-<span class="sourceLineNo">1071</span> }<a name="line.1071"></a>
-<span class="sourceLineNo">1072</span><a name="line.1072"></a>
-<span class="sourceLineNo">1073</span> // Add the procedure to the executor<a name="line.1073"></a>
-<span class="sourceLineNo">1074</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1074"></a>
-<span class="sourceLineNo">1075</span> pushProcedure(procs[i]);<a name="line.1075"></a>
-<span class="sourceLineNo">1076</span> }<a name="line.1076"></a>
-<span class="sourceLineNo">1077</span> }<a name="line.1077"></a>
-<span class="sourceLineNo">1078</span><a name="line.1078"></a>
-<span class="sourceLineNo">1079</span> private Procedure<TEnvironment> prepareProcedure(Procedure<TEnvironment> proc) {<a name="line.1079"></a>
-<span class="sourceLineNo">1080</span> Preconditions.checkArgument(proc.getState() == ProcedureState.INITIALIZING);<a name="line.1080"></a>
-<span class="sourceLineNo">1081</span> Preconditions.checkArgument(!proc.hasParent(), "unexpected parent", proc);<a name="line.1081"></a>
-<span class="sourceLineNo">1082</span> if (this.checkOwnerSet) {<a name="line.1082"></a>
-<span class="sourceLineNo">1083</span> Preconditions.checkArgument(proc.hasOwner(), "missing owner");<a name="line.1083"></a>
-<span class="sourceLineNo">1084</span> }<a name="line.1084"></a>
-<span class="sourceLineNo">1085</span> return proc;<a name="line.1085"></a>
-<span class="sourceLineNo">1086</span> }<a name="line.1086"></a>
-<span class="sourceLineNo">1087</span><a name="line.1087"></a>
-<span class="sourceLineNo">1088</span> private long pushProcedure(Procedure<TEnvironment> proc) {<a name="line.1088"></a>
-<span class="sourceLineNo">1089</span> final long currentProcId = proc.getProcId();<a name="line.1089"></a>
-<span class="sourceLineNo">1090</span><a name="line.1090"></a>
-<span class="sourceLineNo">1091</span> // Update metrics on start of a procedure<a name="line.1091"></a>
-<span class="sourceLineNo">1092</span> proc.updateMetricsOnSubmit(getEnvironment());<a name="line.1092"></a>
-<span class="sourceLineNo">1093</span><a name="line.1093"></a>
-<span class="sourceLineNo">1094</span> // Create the rollback stack for the procedure<a name="line.1094"></a>
-<span class="sourceLineNo">1095</span> RootProcedureState<TEnvironment> stack = new RootProcedureState<>();<a name="line.1095"></a>
-<span class="sourceLineNo">1096</span> rollbackStack.put(currentProcId, stack);<a name="line.1096"></a>
+<span class="sourceLineNo">1017</span> } finally {<a name="line.1017"></a>
+<span class="sourceLineNo">1018</span> if (lockEntry != null) {<a name="line.1018"></a>
+<span class="sourceLineNo">1019</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1019"></a>
+<span class="sourceLineNo">1020</span> }<a name="line.1020"></a>
+<span class="sourceLineNo">1021</span> }<a name="line.1021"></a>
+<span class="sourceLineNo">1022</span> }<a name="line.1022"></a>
+<span class="sourceLineNo">1023</span><a name="line.1023"></a>
+<span class="sourceLineNo">1024</span> /**<a name="line.1024"></a>
+<span class="sourceLineNo">1025</span> * Add a new root-procedure to the executor.<a name="line.1025"></a>
+<span class="sourceLineNo">1026</span> * @param proc the new procedure to execute.<a name="line.1026"></a>
+<span class="sourceLineNo">1027</span> * @param nonceKey the registered unique identifier for this operation from the client or process.<a name="line.1027"></a>
+<span class="sourceLineNo">1028</span> * @return the procedure id, that can be used to monitor the operation<a name="line.1028"></a>
+<span class="sourceLineNo">1029</span> */<a name="line.1029"></a>
+<span class="sourceLineNo">1030</span> @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH",<a name="line.1030"></a>
+<span class="sourceLineNo">1031</span> justification = "FindBugs is blind to the check-for-null")<a name="line.1031"></a>
+<span class="sourceLineNo">1032</span> public long submitProcedure(Procedure<TEnvironment> proc, NonceKey nonceKey) {<a name="line.1032"></a>
+<span class="sourceLineNo">1033</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1033"></a>
+<span class="sourceLineNo">1034</span><a name="line.1034"></a>
+<span class="sourceLineNo">1035</span> prepareProcedure(proc);<a name="line.1035"></a>
+<span class="sourceLineNo">1036</span><a name="line.1036"></a>
+<span class="sourceLineNo">1037</span> final Long currentProcId;<a name="line.1037"></a>
+<span class="sourceLineNo">1038</span> if (nonceKey != null) {<a name="line.1038"></a>
+<span class="sourceLineNo">1039</span> currentProcId = nonceKeysToProcIdsMap.get(nonceKey);<a name="line.1039"></a>
+<span class="sourceLineNo">1040</span> Preconditions.checkArgument(currentProcId != null,<a name="line.1040"></a>
+<span class="sourceLineNo">1041</span> "Expected nonceKey=" + nonceKey + " to be reserved, use registerNonce(); proc=" + proc);<a name="line.1041"></a>
+<span class="sourceLineNo">1042</span> } else {<a name="line.1042"></a>
+<span class="sourceLineNo">1043</span> currentProcId = nextProcId();<a name="line.1043"></a>
+<span class="sourceLineNo">1044</span> }<a name="line.1044"></a>
+<span class="sourceLineNo">1045</span><a name="line.1045"></a>
+<span class="sourceLineNo">1046</span> // Initialize the procedure<a name="line.1046"></a>
+<span class="sourceLineNo">1047</span> proc.setNonceKey(nonceKey);<a name="line.1047"></a>
+<span class="sourceLineNo">1048</span> proc.setProcId(currentProcId.longValue());<a name="line.1048"></a>
+<span class="sourceLineNo">1049</span><a name="line.1049"></a>
+<span class="sourceLineNo">1050</span> // Commit the transaction<a name="line.1050"></a>
+<span class="sourceLineNo">1051</span> store.insert(proc, null);<a name="line.1051"></a>
+<span class="sourceLineNo">1052</span> LOG.debug("Stored {}", proc);<a name="line.1052"></a>
+<span class="sourceLineNo">1053</span><a name="line.1053"></a>
+<span class="sourceLineNo">1054</span> // Add the procedure to the executor<a name="line.1054"></a>
+<span class="sourceLineNo">1055</span> return pushProcedure(proc);<a name="line.1055"></a>
+<span class="sourceLineNo">1056</span> }<a name="line.1056"></a>
+<span class="sourceLineNo">1057</span><a name="line.1057"></a>
+<span class="sourceLineNo">1058</span> /**<a name="line.1058"></a>
+<span class="sourceLineNo">1059</span> * Add a set of new root-procedure to the executor.<a name="line.1059"></a>
+<span class="sourceLineNo">1060</span> * @param procs the new procedures to execute.<a name="line.1060"></a>
+<span class="sourceLineNo">1061</span> */<a name="line.1061"></a>
+<span class="sourceLineNo">1062</span> // TODO: Do we need to take nonces here?<a name="line.1062"></a>
+<span class="sourceLineNo">1063</span> public void submitProcedures(Procedure<TEnvironment>[] procs) {<a name="line.1063"></a>
+<span class="sourceLineNo">1064</span> Preconditions.checkArgument(lastProcId.get() >= 0);<a name="line.1064"></a>
+<span class="sourceLineNo">1065</span> if (procs == null || procs.length <= 0) {<a name="line.1065"></a>
+<span class="sourceLineNo">1066</span> return;<a name="line.1066"></a>
+<span class="sourceLineNo">1067</span> }<a name="line.1067"></a>
+<span class="sourceLineNo">1068</span><a name="line.1068"></a>
+<span class="sourceLineNo">1069</span> // Prepare procedure<a name="line.1069"></a>
+<span class="sourceLineNo">1070</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1070"></a>
+<span class="sourceLineNo">1071</span> prepareProcedure(procs[i]).setProcId(nextProcId());<a name="line.1071"></a>
+<span class="sourceLineNo">1072</span> }<a name="line.1072"></a>
+<span class="sourceLineNo">1073</span><a name="line.1073"></a>
+<span class="sourceLineNo">1074</span> // Commit the transaction<a name="line.1074"></a>
+<span class="sourceLineNo">1075</span> store.insert(procs);<a name="line.1075"></a>
+<span class="sourceLineNo">1076</span> if (LOG.isDebugEnabled()) {<a name="line.1076"></a>
+<span class="sourceLineNo">1077</span> LOG.debug("Stored " + Arrays.toString(procs));<a name="line.1077"></a>
+<span class="sourceLineNo">1078</span> }<a name="line.1078"></a>
+<span class="sourceLineNo">1079</span><a name="line.1079"></a>
+<span class="sourceLineNo">1080</span> // Add the procedure to the executor<a name="line.1080"></a>
+<span class="sourceLineNo">1081</span> for (int i = 0; i < procs.length; ++i) {<a name="line.1081"></a>
+<span class="sourceLineNo">1082</span> pushProcedure(procs[i]);<a name="line.1082"></a>
+<span class="sourceLineNo">1083</span> }<a name="line.1083"></a>
+<span class="sourceLineNo">1084</span> }<a name="line.1084"></a>
+<span class="sourceLineNo">1085</span><a name="line.1085"></a>
+<span class="sourceLineNo">1086</span> private Procedure<TEnvironment> prepareProcedure(Procedure<TEnvironment> proc) {<a name="line.1086"></a>
+<span class="sourceLineNo">1087</span> Preconditions.checkArgument(proc.getState() == ProcedureState.INITIALIZING);<a name="line.1087"></a>
+<span class="sourceLineNo">1088</span> Preconditions.checkArgument(!proc.hasParent(), "unexpected parent", proc);<a name="line.1088"></a>
+<span class="sourceLineNo">1089</span> if (this.checkOwnerSet) {<a name="line.1089"></a>
+<span class="sourceLineNo">1090</span> Preconditions.checkArgument(proc.hasOwner(), "missing owner");<a name="line.1090"></a>
+<span class="sourceLineNo">1091</span> }<a name="line.1091"></a>
+<span class="sourceLineNo">1092</span> return proc;<a name="line.1092"></a>
+<span class="sourceLineNo">1093</span> }<a name="line.1093"></a>
+<span class="sourceLineNo">1094</span><a name="line.1094"></a>
+<span class="sourceLineNo">1095</span> private long pushProcedure(Procedure<TEnvironment> proc) {<a name="line.1095"></a>
+<span class="sourceLineNo">1096</span> final long currentProcId = proc.getProcId();<a name="line.1096"></a>
<span class="sourceLineNo">1097</span><a name="line.1097"></a>
-<span class="sourceLineNo">1098</span> // Submit the new subprocedures<a name="line.1098"></a>
-<span class="sourceLineNo">1099</span> assert !procedures.containsKey(currentProcId);<a name="line.1099"></a>
-<span class="sourceLineNo">1100</span> procedures.put(currentProcId, proc);<a name="line.1100"></a>
-<span class="sourceLineNo">1101</span> sendProcedureAddedNotification(currentProcId);<a name="line.1101"></a>
-<span class="sourceLineNo">1102</span> scheduler.addBack(proc);<a name="line.1102"></a>
-<span class="sourceLineNo">1103</span> return proc.getProcId();<a name="line.1103"></a>
-<span class="sourceLineNo">1104</span> }<a name="line.1104"></a>
-<span class="sourceLineNo">1105</span><a name="line.1105"></a>
-<span class="sourceLineNo">1106</span> /**<a name="line.1106"></a>
-<span class="sourceLineNo">1107</span> * Send an abort notification the specified procedure.<a name="line.1107"></a>
-<span class="sourceLineNo">1108</span> * Depending on the procedure implementation the abort can be considered or ignored.<a name="line.1108"></a>
-<span class="sourceLineNo">1109</span> * @param procId the procedure to abort<a name="line.1109"></a>
-<span class="sourceLineNo">1110</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1110"></a>
-<span class="sourceLineNo">1111</span> */<a name="line.1111"></a>
-<span class="sourceLineNo">1112</span> public boolean abort(long procId) {<a name="line.1112"></a>
-<span class="sourceLineNo">1113</span> return abort(procId, true);<a name="line.1113"></a>
-<span class="sourceLineNo">1114</span> }<a name="line.1114"></a>
-<span class="sourceLineNo">1115</span><a name="line.1115"></a>
-<span class="sourceLineNo">1116</span> /**<a name="line.1116"></a>
-<span class="sourceLineNo">1117</span> * Send an abort notification to the specified procedure.<a name="line.1117"></a>
-<span class="sourceLineNo">1118</span> * Depending on the procedure implementation, the abort can be considered or ignored.<a name="line.1118"></a>
-<span class="sourceLineNo">1119</span> * @param procId the procedure to abort<a name="line.1119"></a>
-<span class="sourceLineNo">1120</span> * @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?<a name="line.1120"></a>
-<span class="sourceLineNo">1121</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1121"></a>
-<span class="sourceLineNo">1122</span> */<a name="line.1122"></a>
-<span class="sourceLineNo">1123</span> public boolean abort(long procId, boolean mayInterruptIfRunning) {<a name="line.1123"></a>
-<span class="sourceLineNo">1124</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.1124"></a>
-<span class="sourceLineNo">1125</span> if (proc != null) {<a name="line.1125"></a>
-<span class="sourceLineNo">1126</span> if (!mayInterruptIfRunning && proc.wasExecuted()) {<a name="line.1126"></a>
-<span class="sourceLineNo">1127</span> return false;<a name="line.1127"></a>
-<span class="sourceLineNo">1128</span> }<a name="line.1128"></a>
-<span class="sourceLineNo">1129</span> return proc.abort(getEnvironment());<a name="line.1129"></a>
-<span class="sourceLineNo">1130</span> }<a name="line.1130"></a>
-<span class="sourceLineNo">1131</span> return false;<a name="line.1131"></a>
-<span class="sourceLineNo">1132</span> }<a name="line.1132"></a>
-<span class="sourceLineNo">1133</span><a name="line.1133"></a>
-<span class="sourceLineNo">1134</span> // ==========================================================================<a name="line.1134"></a>
-<span class="sourceLineNo">1135</span> // Executor query helpers<a name="line.1135"></a>
-<span class="sourceLineNo">1136</span> // ==========================================================================<a name="line.1136"></a>
-<span class="sourceLineNo">1137</span> public Procedure<TEnvironment> getProcedure(final long procId) {<a name="line.1137"></a>
-<span class="sourceLineNo">1138</span> return procedures.get(procId);<a name="line.1138"></a>
+<span class="sourceLineNo">1098</span> // Update metrics on start of a procedure<a name="line.1098"></a>
+<span class="sourceLineNo">1099</span> proc.updateMetricsOnSubmit(getEnvironment());<a name="line.1099"></a>
+<span class="sourceLineNo">1100</span><a name="line.1100"></a>
+<span class="sourceLineNo">1101</span> // Create the rollback stack for the procedure<a name="line.1101"></a>
+<span class="sourceLineNo">1102</span> RootProcedureState<TEnvironment> stack = new RootProcedureState<>();<a name="line.1102"></a>
+<span class="sourceLineNo">1103</span> rollbackStack.put(currentProcId, stack);<a name="line.1103"></a>
+<span class="sourceLineNo">1104</span><a name="line.1104"></a>
+<span class="sourceLineNo">1105</span> // Submit the new subprocedures<a name="line.1105"></a>
+<span class="sourceLineNo">1106</span> assert !procedures.containsKey(currentProcId);<a name="line.1106"></a>
+<span class="sourceLineNo">1107</span> procedures.put(currentProcId, proc);<a name="line.1107"></a>
+<span class="sourceLineNo">1108</span> sendProcedureAddedNotification(currentProcId);<a name="line.1108"></a>
+<span class="sourceLineNo">1109</span> scheduler.addBack(proc);<a name="line.1109"></a>
+<span class="sourceLineNo">1110</span> return proc.getProcId();<a name="line.1110"></a>
+<span class="sourceLineNo">1111</span> }<a name="line.1111"></a>
+<span class="sourceLineNo">1112</span><a name="line.1112"></a>
+<span class="sourceLineNo">1113</span> /**<a name="line.1113"></a>
+<span class="sourceLineNo">1114</span> * Send an abort notification the specified procedure.<a name="line.1114"></a>
+<span class="sourceLineNo">1115</span> * Depending on the procedure implementation the abort can be considered or ignored.<a name="line.1115"></a>
+<span class="sourceLineNo">1116</span> * @param procId the procedure to abort<a name="line.1116"></a>
+<span class="sourceLineNo">1117</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1117"></a>
+<span class="sourceLineNo">1118</span> */<a name="line.1118"></a>
+<span class="sourceLineNo">1119</span> public boolean abort(long procId) {<a name="line.1119"></a>
+<span class="sourceLineNo">1120</span> return abort(procId, true);<a name="line.1120"></a>
+<span class="sourceLineNo">1121</span> }<a name="line.1121"></a>
+<span class="sourceLineNo">1122</span><a name="line.1122"></a>
+<span class="sourceLineNo">1123</span> /**<a name="line.1123"></a>
+<span class="sourceLineNo">1124</span> * Send an abort notification to the specified procedure.<a name="line.1124"></a>
+<span class="sourceLineNo">1125</span> * Depending on the procedure implementation, the abort can be considered or ignored.<a name="line.1125"></a>
+<span class="sourceLineNo">1126</span> * @param procId the procedure to abort<a name="line.1126"></a>
+<span class="sourceLineNo">1127</span> * @param mayInterruptIfRunning if the proc completed at least one step, should it be aborted?<a name="line.1127"></a>
+<span class="sourceLineNo">1128</span> * @return true if the procedure exists and has received the abort, otherwise false.<a name="line.1128"></a>
+<span class="sourceLineNo">1129</span> */<a name="line.1129"></a>
+<span class="sourceLineNo">1130</span> public boolean abort(long procId, boolean mayInterruptIfRunning) {<a name="line.1130"></a>
+<span class="sourceLineNo">1131</span> Procedure<TEnvironment> proc = procedures.get(procId);<a name="line.1131"></a>
+<span class="sourceLineNo">1132</span> if (proc != null) {<a name="line.1132"></a>
+<span class="sourceLineNo">1133</span> if (!mayInterruptIfRunning && proc.wasExecuted()) {<a name="line.1133"></a>
+<span class="sourceLineNo">1134</span> return false;<a name="line.1134"></a>
+<span class="sourceLineNo">1135</span> }<a name="line.1135"></a>
+<span class="sourceLineNo">1136</span> return proc.abort(getEnvironment());<a name="line.1136"></a>
+<span class="sourceLineNo">1137</span> }<a name="line.1137"></a>
+<span class="sourceLineNo">1138</span> return false;<a name="line.1138"></a>
<span class="sourceLineNo">1139</span> }<a name="line.1139"></a>
<span class="sourceLineNo">1140</span><a name="line.1140"></a>
-<span class="sourceLineNo">1141</span> public <T extends Procedure<TEnvironment>> T getProcedure(Class<T> clazz, long procId) {<a name="line.1141"></a>
-<span class="sourceLineNo">1142</span> Procedure<TEnvironment> proc = getProcedure(procId);<a name="line.1142"></a>
-<span class="sourceLineNo">1143</span> if (clazz.isInstance(proc)) {<a name="line.1143"></a>
-<span class="sourceLineNo">1144</span> return clazz.cast(proc);<a name="line.1144"></a>
-<span class="sourceLineNo">1145</span> }<a name="line.1145"></a>
-<span class="sourceLineNo">1146</span> return null;<a name="line.1146"></a>
-<span class="sourceLineNo">1147</span> }<a name="line.1147"></a>
-<span class="sourceLineNo">1148</span><a name="line.1148"></a>
-<span class="sourceLineNo">1149</span> public Procedure<TEnvironment> getResult(long procId) {<a name="line.1149"></a>
-<span class="sourceLineNo">1150</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1150"></a>
-<span class="sourceLineNo">1151</span> if (retainer == null) {<a name="line.1151"></a>
-<span class="sourceLineNo">1152</span> return null;<a name="line.1152"></a>
-<span class="sourceLineNo">1153</span> } else {<a name="line.1153"></a>
-<span class="sourceLineNo">1154</span> return retainer.getProcedure();<a name="line.1154"></a>
-<span class="sourceLineNo">1155</span> }<a name="line.1155"></a>
-<span class="sourceLineNo">1156</span> }<a name="line.1156"></a>
-<span class="sourceLineNo">1157</span><a name="line.1157"></a>
-<span class="sourceLineNo">1158</span> /**<a name="line.1158"></a>
-<span class="sourceLineNo">1159</span> * Return true if the procedure is finished.<a name="line.1159"></a>
-<span class="sourceLineNo">1160</span> * The state may be "completed successfully" or "failed and rolledback".<a name="line.1160"></a>
-<span class="sourceLineNo">1161</span> * Use getResult() to check the state or get the result data.<a name="line.1161"></a>
-<span class="sourceLineNo">1162</span> * @param procId the ID of the procedure to check<a name="line.1162"></a>
-<span class="sourceLineNo">1163</span> * @return true if the procedure execution is finished, otherwise false.<a name="line.1163"></a>
-<span class="sourceLineNo">1164</span> */<a name="line.1164"></a>
-<span class="sourceLineNo">1165</span> public boolean isFinished(final long procId) {<a name="line.1165"></a>
-<span class="sourceLineNo">1166</span> return !procedures.containsKey(procId);<a name="line.1166"></a>
-<span class="sourceLineNo">1167</span> }<a name="line.1167"></a>
-<span class="sourceLineNo">1168</span><a name="line.1168"></a>
-<span class="sourceLineNo">1169</span> /**<a name="line.1169"></a>
-<span class="sourceLineNo">1170</span> * Return true if the procedure is started.<a name="line.1170"></a>
-<span class="sourceLineNo">1171</span> * @param procId the ID of the procedure to check<a name="line.1171"></a>
-<span class="sourceLineNo">1172</span> * @return true if the procedure execution is started, otherwise false.<a name="line.1172"></a>
-<span class="sourceLineNo">1173</span> */<a name="line.1173"></a>
-<span class="sourceLineNo">1174</span> public boolean isStarted(long procId) {<a name="line.1174"></a>
-<span class="sourceLineNo">1175</span> Procedure<?> proc = procedures.get(procId);<a name="line.1175"></a>
-<span class="sourceLineNo">1176</span> if (proc == null) {<a name="line.1176"></a>
-<span class="sourceLineNo">1177</span> return completed.get(procId) != null;<a name="line.1177"></a>
-<span class="sourceLineNo">1178</span> }<a name="line.1178"></a>
-<span class="sourceLineNo">1179</span> return proc.wasExecuted();<a name="line.1179"></a>
-<span class="sourceLineNo">1180</span> }<a name="line.1180"></a>
-<span class="sourceLineNo">1181</span><a name="line.1181"></a>
-<span class="sourceLineNo">1182</span> /**<a name="line.1182"></a>
-<span class="sourceLineNo">1183</span> * Mark the specified completed procedure, as ready to remove.<a name="line.1183"></a>
-<span class="sourceLineNo">1184</span> * @param procId the ID of the procedure to remove<a name="line.1184"></a>
-<span class="sourceLineNo">1185</span> */<a name="line.1185"></a>
-<span class="sourceLineNo">1186</span> public void removeResult(long procId) {<a name="line.1186"></a>
-<span class="sourceLineNo">1187</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1187"></a>
-<span class="sourceLineNo">1188</span> if (retainer == null) {<a name="line.1188"></a>
-<span class="sourceLineNo">1189</span> assert !procedures.containsKey(procId) : "pid=" + procId + " is still running";<a name="line.1189"></a>
-<span class="sourceLineNo">1190</span> LOG.debug("pid={} already removed by the cleaner.", procId);<a name="line.1190"></a>
-<span class="sourceLineNo">1191</span> return;<a name="line.1191"></a>
-<span class="sourceLineNo">1192</span> }<a name="line.1192"></a>
-<span class="sourceLineNo">1193</span><a name="line.1193"></a>
-<span class="sourceLineNo">1194</span> // The CompletedProcedureCleaner will take care of deletion, once the TTL is expired.<a name="line.1194"></a>
-<span class="sourceLineNo">1195</span> retainer.setClientAckTime(EnvironmentEdgeManager.currentTime());<a name="line.1195"></a>
-<span class="sourceLineNo">1196</span> }<a name="line.1196"></a>
-<span class="sourceLineNo">1197</span><a name="line.1197"></a>
-<span class="sourceLineNo">1198</span> public Procedure<TEnvironment> getResultOrProcedure(long procId) {<a name="line.1198"></a>
-<span class="sourceLineNo">1199</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1199"></a>
-<span class="sourceLineNo">1200</span> if (retainer == null) {<a name="line.1200"></a>
-<span class="sourceLineNo">1201</span> return procedures.get(procId);<a name="line.1201"></a>
-<span class="sourceLineNo">1202</span> } else {<a name="line.1202"></a>
-<span class="sourceLineNo">1203</span> return retainer.getProcedure();<a name="line.1203"></a>
-<span class="sourceLineNo">1204</span> }<a name="line.1204"></a>
-<span class="sourceLineNo">1205</span> }<a name="line.1205"></a>
-<span class="sourceLineNo">1206</span><a name="line.1206"></a>
-<span class="sourceLineNo">1207</span> /**<a name="line.1207"></a>
-<span class="sourceLineNo">1208</span> * Check if the user is this procedure's owner<a name="line.1208"></a>
-<span class="sourceLineNo">1209</span> * @param procId the target procedure<a name="line.1209"></a>
-<span class="sourceLineNo">1210</span> * @param user the user<a name="line.1210"></a>
-<span class="sourceLineNo">1211</span> * @return true if the user is the owner of the procedure,<a name="line.1211"></a>
-<span class="sourceLineNo">1212</span> * false otherwise or the owner is unknown.<a name="line.1212"></a>
-<span class="sourceLineNo">1213</span> */<a name="line.1213"></a>
-<span class="sourceLineNo">1214</span> public boolean isProcedureOwner(long procId, User user) {<a name="line.1214"></a>
-<span class="sourceLineNo">1215</span> if (user == null) {<a name="line.1215"></a>
-<span class="sourceLineNo">1216</span> return false;<a name="line.1216"></a>
-<span class="sourceLineNo">1217</span> }<a name="line.1217"></a>
-<span class="sourceLineNo">1218</span> final Procedure<TEnvironment> runningProc = procedures.get(procId);<a name="line.1218"></a>
-<span class="sourceLineNo">1219</span> if (runningProc != null) {<a name="line.1219"></a>
-<span class="sourceLineNo">1220</span> return runningProc.getOwner().equals(user.getShortName());<a name="line.1220"></a>
-<span class="sourceLineNo">1221</span> }<a name="line.1221"></a>
-<span class="sourceLineNo">1222</span><a name="line.1222"></a>
-<span class="sourceLineNo">1223</span> final CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1223"></a>
-<span class="sourceLineNo">1224</span> if (retainer != null) {<a name="line.1224"></a>
-<span class="sourceLineNo">1225</span> return retainer.getProcedure().getOwner().equals(user.getShortName());<a name="line.1225"></a>
-<span class="sourceLineNo">1226</span> }<a name="line.1226"></a>
-<span class="sourceLineNo">1227</span><a name="line.1227"></a>
-<span class="sourceLineNo">1228</span> // Procedure either does not exist or has already completed and got cleaned up.<a name="line.1228"></a>
-<span class="sourceLineNo">1229</span> // At this time, we cannot check the owner of the procedure<a name="line.1229"></a>
-<span class="sourceLineNo">1230</span> return false;<a name="line.1230"></a>
-<span class="sourceLineNo">1231</span> }<a name="line.1231"></a>
-<span class="sourceLineNo">1232</span><a name="line.1232"></a>
-<span class="sourceLineNo">1233</span> /**<a name="line.1233"></a>
-<span class="sourceLineNo">1234</span> * Should only be used when starting up, where the procedure workers have not been started.<a name="line.1234"></a>
-<span class="sourceLineNo">1235</span> * <p/><a name="line.1235"></a>
-<span class="sourceLineNo">1236</span> * If the procedure works has been started, the return values maybe changed when you are<a name="line.1236"></a>
-<span class="sourceLineNo">1237</span> * processing it so usually this is not safe. Use {@link #getProcedures()} below for most cases as<a name="line.1237"></a>
-<span class="sourceLineNo">1238</span> * it will do a copy, and also include the finished procedures.<a name="line.1238"></a>
-<span class="sourceLineNo">1239</span> */<a name="line.1239"></a>
-<span class="sourceLineNo">1240</span> public Collection<Procedure<TEnvironment>> getActiveProceduresNoCopy() {<a name="line.1240"></a>
-<span class="sourceLineNo">1241</span> return procedures.values();<a name="line.1241"></a>
-<span class="sourceLineNo">1242</span> }<a name="line.1242"></a>
-<span class="sourceLineNo">1243</span><a name="line.1243"></a>
-<span class="sourceLineNo">1244</span> /**<a name="line.1244"></a>
-<span class="sourceLineNo">1245</span> * Get procedures.<a name="line.1245"></a>
-<span class="sourceLineNo">1246</span> * @return the procedures in a list<a name="line.1246"></a>
-<span class="sourceLineNo">1247</span> */<a name="line.1247"></a>
-<span class="sourceLineNo">1248</span> public List<Procedure<TEnvironment>> getProcedures() {<a name="line.1248"></a>
-<span class="sourceLineNo">1249</span> List<Procedure<TEnvironment>> procedureList =<a name="line.1249"></a>
-<span class="sourceLineNo">1250</span> new ArrayList<>(procedures.size() + completed.size());<a name="line.1250"></a>
-<span class="sourceLineNo">1251</span> procedureList.addAll(procedures.values());<a name="line.1251"></a>
-<span class="sourceLineNo">1252</span> // Note: The procedure could show up twice in the list with different state, as<a name="line.1252"></a>
-<span class="sourceLineNo">1253</span> // it could complete after we walk through procedures list and insert into<a name="line.1253"></a>
-<span class="sourceLineNo">1254</span> // procedureList - it is ok, as we will use the information in the Procedure<a name="line.1254"></a>
-<span class="sourceLineNo">1255</span> // to figure it out; to prevent this would increase the complexity of the logic.<a name="line.1255"></a>
-<span class="sourceLineNo">1256</span> completed.values().stream().map(CompletedProcedureRetainer::getProcedure)<a name="line.1256"></a>
-<span class="sourceLineNo">1257</span> .forEach(procedureList::add);<a name="line.1257"></a>
-<span class="sourceLineNo">1258</span> return procedureList;<a name="line.1258"></a>
-<span class="sourceLineNo">1259</span> }<a name="line.1259"></a>
-<span class="sourceLineNo">1260</span><a name="line.1260"></a>
-<span class="sourceLineNo">1261</span> // ==========================================================================<a name="line.1261"></a>
-<span class="sourceLineNo">1262</span> // Listeners helpers<a name="line.1262"></a>
-<span class="sourceLineNo">1263</span> // ==========================================================================<a name="line.1263"></a>
-<span class="sourceLineNo">1264</span> public void registerListener(ProcedureExecutorListener listener) {<a name="line.1264"></a>
-<span class="sourceLineNo">1265</span> this.listeners.add(listener);<a name="line.1265"></a>
+<span class="sourceLineNo">1141</span> // ==========================================================================<a name="line.1141"></a>
+<span class="sourceLineNo">1142</span> // Executor query helpers<a name="line.1142"></a>
+<span class="sourceLineNo">1143</span> // ==========================================================================<a name="line.1143"></a>
+<span class="sourceLineNo">1144</span> public Procedure<TEnvironment> getProcedure(final long procId) {<a name="line.1144"></a>
+<span class="sourceLineNo">1145</span> return procedures.get(procId);<a name="line.1145"></a>
+<span class="sourceLineNo">1146</span> }<a name="line.1146"></a>
+<span class="sourceLineNo">1147</span><a name="line.1147"></a>
+<span class="sourceLineNo">1148</span> public <T extends Procedure<TEnvironment>> T getProcedure(Class<T> clazz, long procId) {<a name="line.1148"></a>
+<span class="sourceLineNo">1149</span> Procedure<TEnvironment> proc = getProcedure(procId);<a name="line.1149"></a>
+<span class="sourceLineNo">1150</span> if (clazz.isInstance(proc)) {<a name="line.1150"></a>
+<span class="sourceLineNo">1151</span> return clazz.cast(proc);<a name="line.1151"></a>
+<span class="sourceLineNo">1152</span> }<a name="line.1152"></a>
+<span class="sourceLineNo">1153</span> return null;<a name="line.1153"></a>
+<span class="sourceLineNo">1154</span> }<a name="line.1154"></a>
+<span class="sourceLineNo">1155</span><a name="line.1155"></a>
+<span class="sourceLineNo">1156</span> public Procedure<TEnvironment> getResult(long procId) {<a name="line.1156"></a>
+<span class="sourceLineNo">1157</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1157"></a>
+<span class="sourceLineNo">1158</span> if (retainer == null) {<a name="line.1158"></a>
+<span class="sourceLineNo">1159</span> return null;<a name="line.1159"></a>
+<span class="sourceLineNo">1160</span> } else {<a name="line.1160"></a>
+<span class="sourceLineNo">1161</span> return retainer.getProcedure();<a name="line.1161"></a>
+<span class="sourceLineNo">1162</span> }<a name="line.1162"></a>
+<span class="sourceLineNo">1163</span> }<a name="line.1163"></a>
+<span class="sourceLineNo">1164</span><a name="line.1164"></a>
+<span class="sourceLineNo">1165</span> /**<a name="line.1165"></a>
+<span class="sourceLineNo">1166</span> * Return true if the procedure is finished.<a name="line.1166"></a>
+<span class="sourceLineNo">1167</span> * The state may be "completed successfully" or "failed and rolledback".<a name="line.1167"></a>
+<span class="sourceLineNo">1168</span> * Use getResult() to check the state or get the result data.<a name="line.1168"></a>
+<span class="sourceLineNo">1169</span> * @param procId the ID of the procedure to check<a name="line.1169"></a>
+<span class="sourceLineNo">1170</span> * @return true if the procedure execution is finished, otherwise false.<a name="line.1170"></a>
+<span class="sourceLineNo">1171</span> */<a name="line.1171"></a>
+<span class="sourceLineNo">1172</span> public boolean isFinished(final long procId) {<a name="line.1172"></a>
+<span class="sourceLineNo">1173</span> return !procedures.containsKey(procId);<a name="line.1173"></a>
+<span class="sourceLineNo">1174</span> }<a name="line.1174"></a>
+<span class="sourceLineNo">1175</span><a name="line.1175"></a>
+<span class="sourceLineNo">1176</span> /**<a name="line.1176"></a>
+<span class="sourceLineNo">1177</span> * Return true if the procedure is started.<a name="line.1177"></a>
+<span class="sourceLineNo">1178</span> * @param procId the ID of the procedure to check<a name="line.1178"></a>
+<span class="sourceLineNo">1179</span> * @return true if the procedure execution is started, otherwise false.<a name="line.1179"></a>
+<span class="sourceLineNo">1180</span> */<a name="line.1180"></a>
+<span class="sourceLineNo">1181</span> public boolean isStarted(long procId) {<a name="line.1181"></a>
+<span class="sourceLineNo">1182</span> Procedure<?> proc = procedures.get(procId);<a name="line.1182"></a>
+<span class="sourceLineNo">1183</span> if (proc == null) {<a name="line.1183"></a>
+<span class="sourceLineNo">1184</span> return completed.get(procId) != null;<a name="line.1184"></a>
+<span class="sourceLineNo">1185</span> }<a name="line.1185"></a>
+<span class="sourceLineNo">1186</span> return proc.wasExecuted();<a name="line.1186"></a>
+<span class="sourceLineNo">1187</span> }<a name="line.1187"></a>
+<span class="sourceLineNo">1188</span><a name="line.1188"></a>
+<span class="sourceLineNo">1189</span> /**<a name="line.1189"></a>
+<span class="sourceLineNo">1190</span> * Mark the specified completed procedure, as ready to remove.<a name="line.1190"></a>
+<span class="sourceLineNo">1191</span> * @param procId the ID of the procedure to remove<a name="line.1191"></a>
+<span class="sourceLineNo">1192</span> */<a name="line.1192"></a>
+<span class="sourceLineNo">1193</span> public void removeResult(long procId) {<a name="line.1193"></a>
+<span class="sourceLineNo">1194</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1194"></a>
+<span class="sourceLineNo">1195</span> if (retainer == null) {<a name="line.1195"></a>
+<span class="sourceLineNo">1196</span> assert !procedures.containsKey(procId) : "pid=" + procId + " is still running";<a name="line.1196"></a>
+<span class="sourceLineNo">1197</span> LOG.debug("pid={} already removed by the cleaner.", procId);<a name="line.1197"></a>
+<span class="sourceLineNo">1198</span> return;<a name="line.1198"></a>
+<span class="sourceLineNo">1199</span> }<a name="line.1199"></a>
+<span class="sourceLineNo">1200</span><a name="line.1200"></a>
+<span class="sourceLineNo">1201</span> // The CompletedProcedureCleaner will take care of deletion, once the TTL is expired.<a name="line.1201"></a>
+<span class="sourceLineNo">1202</span> retainer.setClientAckTime(EnvironmentEdgeManager.currentTime());<a name="line.1202"></a>
+<span class="sourceLineNo">1203</span> }<a name="line.1203"></a>
+<span class="sourceLineNo">1204</span><a name="line.1204"></a>
+<span class="sourceLineNo">1205</span> public Procedure<TEnvironment> getResultOrProcedure(long procId) {<a name="line.1205"></a>
+<span class="sourceLineNo">1206</span> CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1206"></a>
+<span class="sourceLineNo">1207</span> if (retainer == null) {<a name="line.1207"></a>
+<span class="sourceLineNo">1208</span> return procedures.get(procId);<a name="line.1208"></a>
+<span class="sourceLineNo">1209</span> } else {<a name="line.1209"></a>
+<span class="sourceLineNo">1210</span> return retainer.getProcedure();<a name="line.1210"></a>
+<span class="sourceLineNo">1211</span> }<a name="line.1211"></a>
+<span class="sourceLineNo">1212</span> }<a name="line.1212"></a>
+<span class="sourceLineNo">1213</span><a name="line.1213"></a>
+<span class="sourceLineNo">1214</span> /**<a name="line.1214"></a>
+<span class="sourceLineNo">1215</span> * Check if the user is this procedure's owner<a name="line.1215"></a>
+<span class="sourceLineNo">1216</span> * @param procId the target procedure<a name="line.1216"></a>
+<span class="sourceLineNo">1217</span> * @param user the user<a name="line.1217"></a>
+<span class="sourceLineNo">1218</span> * @return true if the user is the owner of the procedure,<a name="line.1218"></a>
+<span class="sourceLineNo">1219</span> * false otherwise or the owner is unknown.<a name="line.1219"></a>
+<span class="sourceLineNo">1220</span> */<a name="line.1220"></a>
+<span class="sourceLineNo">1221</span> public boolean isProcedureOwner(long procId, User user) {<a name="line.1221"></a>
+<span class="sourceLineNo">1222</span> if (user == null) {<a name="line.1222"></a>
+<span class="sourceLineNo">1223</span> return false;<a name="line.1223"></a>
+<span class="sourceLineNo">1224</span> }<a name="line.1224"></a>
+<span class="sourceLineNo">1225</span> final Procedure<TEnvironment> runningProc = procedures.get(procId);<a name="line.1225"></a>
+<span class="sourceLineNo">1226</span> if (runningProc != null) {<a name="line.1226"></a>
+<span class="sourceLineNo">1227</span> return runningProc.getOwner().equals(user.getShortName());<a name="line.1227"></a>
+<span class="sourceLineNo">1228</span> }<a name="line.1228"></a>
+<span class="sourceLineNo">1229</span><a name="line.1229"></a>
+<span class="sourceLineNo">1230</span> final CompletedProcedureRetainer<TEnvironment> retainer = completed.get(procId);<a name="line.1230"></a>
+<span class="sourceLineNo">1231</span> if (retainer != null) {<a name="line.1231"></a>
+<span class="sourceLineNo">1232</span> return retainer.getProcedure().getOwner().equals(user.getShortName());<a name="line.1232"></a>
+<span class="sourceLineNo">1233</span> }<a name="line.1233"></a>
+<span class="sourceLineNo">1234</span><a name="line.1234"></a>
+<span class="sourceLineNo">1235</span> // Procedure either does not exist or has already completed and got cleaned up.<a name="line.1235"></a>
+<span class="sourceLineNo">1236</span> // At this time, we cannot check the owner of the procedure<a name="line.1236"></a>
+<span class="sourceLineNo">1237</span> return false;<a name="line.1237"></a>
+<span class="sourceLineNo">1238</span> }<a name="line.1238"></a>
+<span class="sourceLineNo">1239</span><a name="line.1239"></a>
+<span class="sourceLineNo">1240</span> /**<a name="line.1240"></a>
+<span class="sourceLineNo">1241</span> * Should only be used when starting up, where the procedure workers have not been started.<a name="line.1241"></a>
+<span class="sourceLineNo">1242</span> * <p/><a name="line.1242"></a>
+<span class="sourceLineNo">1243</span> * If the procedure works has been started, the return values maybe changed when you are<a name="line.1243"></a>
+<span class="sourceLineNo">1244</span> * processing it so usually this is not safe. Use {@link #getProcedures()} below for most cases as<a name="line.1244"></a>
+<span class="sourceLineNo">1245</span> * it will do a copy, and also include the finished procedures.<a name="line.1245"></a>
+<span class="sourceLineNo">1246</span> */<a name="line.1246"></a>
+<span class="sourceLineNo">1247</span> public Collection<Procedure<TEnvironment>> getActiveProceduresNoCopy() {<a name="line.1247"></a>
+<span class="sourceLineNo">1248</span> return procedures.values();<a name="line.1248"></a>
+<span class="sourceLineNo">1249</span> }<a name="line.1249"></a>
+<span class="sourceLineNo">1250</span><a name="line.1250"></a>
+<span class="sourceLineNo">1251</span> /**<a name="line.1251"></a>
+<span class="sourceLineNo">1252</span> * Get procedures.<a name="line.1252"></a>
+<span class="sourceLineNo">1253</span> * @return the procedures in a list<a name="line.1253"></a>
+<span class="sourceLineNo">1254</span> */<a name="line.1254"></a>
+<span class="sourceLineNo">1255</span> public List<Procedure<TEnvironment>> getProcedures() {<a name="line.1255"></a>
+<span class="sourceLineNo">1256</span> List<Procedure<TEnvironment>> procedureList =<a name="line.1256"></a>
+<span class="sourceLineNo">1257</span> new ArrayList<>(procedures.size() + completed.size());<a name="line.1257"></a>
+<span class="sourceLineNo">1258</span> procedureList.addAll(procedures.values());<a name="line.1258"></a>
+<span class="sourceLineNo">1259</span> // Note: The procedure could show up twice in the list with different state, as<a name="line.1259"></a>
+<span class="sourceLineNo">1260</span> // it could complete after we walk through procedures list and insert into<a name="line.1260"></a>
+<span class="sourceLineNo">1261</span> // procedureList - it is ok, as we will use the information in the Procedure<a name="line.1261"></a>
+<span class="sourceLineNo">1262</span> // to figure it out; to prevent this would increase the complexity of the logic.<a name="line.1262"></a>
+<span class="sourceLineNo">1263</span> completed.values().stream().map(CompletedProcedureRetainer::getProcedure)<a name="line.1263"></a>
+<span class="sourceLineNo">1264</span> .forEach(procedureList::add);<a name="line.1264"></a>
+<span class="sourceLineNo">1265</span> return procedureList;<a name="line.1265"></a>
<span class="sourceLineNo">1266</span> }<a name="line.1266"></a>
<span class="sourceLineNo">1267</span><a name="line.1267"></a>
-<span class="sourceLineNo">1268</span> public boolean unregisterListener(ProcedureExecutorListener listener) {<a name="line.1268"></a>
-<span class="sourceLineNo">1269</span> return this.listeners.remove(listener);<a name="line.1269"></a>
-<span class="sourceLineNo">1270</span> }<a name="line.1270"></a>
-<span class="sourceLineNo">1271</span><a name="line.1271"></a>
-<span class="sourceLineNo">1272</span> private void sendProcedureLoadedNotification(final long procId) {<a name="line.1272"></a>
-<span class="sourceLineNo">1273</span> if (!this.listeners.isEmpty()) {<a name="line.1273"></a>
-<span class="sourceLineNo">1274</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1274"></a>
-<span class="sourceLineNo">1275</span> try {<a name="line.1275"></a>
-<span class="sourceLineNo">1276</span> listener.procedureLoaded(procId);<a name="line.1276"></a>
-<span class="sourceLineNo">1277</span> } catch (Throwable e) {<a name="line.1277"></a>
-<span class="sourceLineNo">1278</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1278"></a>
-<span class="sourceLineNo">1279</span> }<a name="line.1279"></a>
-<span class="sourceLineNo">1280</span> }<a name="line.1280"></a>
-<span class="sourceLineNo">1281</span> }<a name="line.1281"></a>
-<span class="sourceLineNo">1282</span> }<a name="line.1282"></a>
-<span class="sourceLineNo">1283</span><a name="line.1283"></a>
-<span class="sourceLineNo">1284</span> private void sendProcedureAddedNotification(final long procId) {<a name="line.1284"></a>
-<span class="sourceLineNo">1285</span> if (!this.listeners.isEmpty()) {<a name="line.1285"></a>
-<span class="sourceLineNo">1286</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1286"></a>
-<span class="sourceLineNo">1287</span> try {<a name="line.1287"></a>
-<span class="sourceLineNo">1288</span> listener.procedureAdded(procId);<a name="line.1288"></a>
-<span class="sourceLineNo">1289</span> } catch (Throwable e) {<a name="line.1289"></a>
-<span class="sourceLineNo">1290</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1290"></a>
-<span class="sourceLineNo">1291</span> }<a name="line.1291"></a>
-<span class="sourceLineNo">1292</span> }<a name="line.1292"></a>
-<span class="sourceLineNo">1293</span> }<a name="line.1293"></a>
-<span class="sourceLineNo">1294</span> }<a name="line.1294"></a>
-<span class="sourceLineNo">1295</span><a name="line.1295"></a>
-<span class="sourceLineNo">1296</span> private void sendProcedureFinishedNotification(final long procId) {<a name="line.1296"></a>
-<span class="sourceLineNo">1297</span> if (!this.listeners.isEmpty()) {<a name="line.1297"></a>
-<span class="sourceLineNo">1298</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1298"></a>
-<span class="sourceLineNo">1299</span> try {<a name="line.1299"></a>
-<span class="sourceLineNo">1300</span> listener.procedureFinished(procId);<a name="line.1300"></a>
-<span class="sourceLineNo">1301</span> } catch (Throwable e) {<a name="line.1301"></a>
-<span class="sourceLineNo">1302</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1302"></a>
-<span class="sourceLineNo">1303</span> }<a name="line.1303"></a>
-<span class="sourceLineNo">1304</span> }<a name="line.1304"></a>
-<span class="sourceLineNo">1305</span> }<a name="line.1305"></a>
-<span class="sourceLineNo">1306</span> }<a name="line.1306"></a>
-<span class="sourceLineNo">1307</span><a name="line.1307"></a>
-<span class="sourceLineNo">1308</span> // ==========================================================================<a name="line.1308"></a>
-<span class="sourceLineNo">1309</span> // Procedure IDs helpers<a name="line.1309"></a>
-<span class="sourceLineNo">1310</span> // ==========================================================================<a name="line.1310"></a>
-<span class="sourceLineNo">1311</span> private long nextProcId() {<a name="line.1311"></a>
-<span class="sourceLineNo">1312</span> long procId = lastProcId.incrementAndGet();<a name="line.1312"></a>
-<span class="sourceLineNo">1313</span> if (procId < 0) {<a name="line.1313"></a>
-<span class="sourceLineNo">1314</span> while (!lastProcId.compareAndSet(procId, 0)) {<a name="line.1314"></a>
-<span class="sourceLineNo">1315</span> procId = lastProcId.get();<a name="line.1315"></a>
-<span class="sourceLineNo">1316</span> if (procId >= 0) {<a name="line.1316"></a>
-<span class="sourceLineNo">1317</span> break;<a name="line.1317"></a>
-<span class="sourceLineNo">1318</span> }<a name="line.1318"></a>
-<span class="sourceLineNo">1319</span> }<a name="line.1319"></a>
-<span class="sourceLineNo">1320</span> while (procedures.containsKey(procId)) {<a name="line.1320"></a>
-<span class="sourceLineNo">1321</span> procId = lastProcId.incrementAndGet();<a name="line.1321"></a>
-<span class="sourceLineNo">1322</span> }<a name="line.1322"></a>
-<span class="sourceLineNo">1323</span> }<a name="line.1323"></a>
-<span class="sourceLineNo">1324</span> assert procId >= 0 : "Invalid procId " + procId;<a name="line.1324"></a>
-<span class="sourceLineNo">1325</span> return procId;<a name="line.1325"></a>
-<span class="sourceLineNo">1326</span> }<a name="line.1326"></a>
-<span class="sourceLineNo">1327</span><a name="line.1327"></a>
-<span class="sourceLineNo">1328</span> @VisibleForTesting<a name="line.1328"></a>
-<span class="sourceLineNo">1329</span> protected long getLastProcId() {<a name="line.1329"></a>
-<span class="sourceLineNo">1330</span> return lastProcId.get();<a name="line.1330"></a>
-<span class="sourceLineNo">1331</span> }<a name="line.1331"></a>
-<span class="sourceLineNo">1332</span><a name="line.1332"></a>
-<span class="sourceLineNo">1333</span> @VisibleForTesting<a name="line.1333"></a>
-<span class="sourceLineNo">1334</span> public Set<Long> getActiveProcIds() {<a name="line.1334"></a>
-<span class="sourceLineNo">1335</span> return procedures.keySet();<a name="line.1335"></a>
-<span class="sourceLineNo">1336</span> }<a name="line.1336"></a>
-<span class="sourceLineNo">1337</span><a name="line.1337"></a>
-<span class="sourceLineNo">1338</span> Long getRootProcedureId(Procedure<TEnvironment> proc) {<a name="line.1338"></a>
-<span class="sourceLineNo">1339</span> return Procedure.getRootProcedureId(procedures, proc);<a name="line.1339"></a>
-<span class="sourceLineNo">1340</span> }<a name="line.1340"></a>
-<span class="sourceLineNo">1341</span><a name="line.1341"></a>
-<span class="sourceLineNo">1342</span> // ==========================================================================<a name="line.1342"></a>
-<span class="sourceLineNo">1343</span> // Executions<a name="line.1343"></a>
-<span class="sourceLineNo">1344</span> // ==========================================================================<a name="line.1344"></a>
-<span class="sourceLineNo">1345</span> private void executeProcedure(Procedure<TEnvironment> proc) {<a name="line.1345"></a>
-<span class="sourceLineNo">1346</span> if (proc.isFinished()) {<a name="line.1346"></a>
-<span class="sourceLineNo">1347</span> LOG.debug("{} is already finished, skipping execution", proc);<a name="line.1347"></a>
-<span class="sourceLineNo">1348</span> return;<a name="line.1348"></a>
-<span class="sourceLineNo">1349</span> }<a name="line.1349"></a>
-<span class="sourceLineNo">1350</span> final Long rootProcId = getRootProcedureId(proc);<a name="line.1350"></a>
-<span class="sourceLineNo">1351</span> if (rootProcId == null) {<a name="line.1351"></a>
-<span class="sourceLineNo">1352</span> // The 'proc' was ready to run but the root procedure was rolledback<a name="line.1352"></a>
-<span class="sourceLineNo">1353</span> LOG.warn("Rollback because parent is done/rolledback proc=" + proc);<a name="line.1353"></a>
-<span class="sourceLineNo">1354</span> executeRollback(proc);<a name="line.1354"></a>
+<span class="sourceLineNo">1268</span> // ==========================================================================<a name="line.1268"></a>
+<span class="sourceLineNo">1269</span> // Listeners helpers<a name="line.1269"></a>
+<span class="sourceLineNo">1270</span> // ==========================================================================<a name="line.1270"></a>
+<span class="sourceLineNo">1271</span> public void registerListener(ProcedureExecutorListener listener) {<a name="line.1271"></a>
+<span class="sourceLineNo">1272</span> this.listeners.add(listener);<a name="line.1272"></a>
+<span class="sourceLineNo">1273</span> }<a name="line.1273"></a>
+<span class="sourceLineNo">1274</span><a name="line.1274"></a>
+<span class="sourceLineNo">1275</span> public boolean unregisterListener(ProcedureExecutorListener listener) {<a name="line.1275"></a>
+<span class="sourceLineNo">1276</span> return this.listeners.remove(listener);<a name="line.1276"></a>
+<span class="sourceLineNo">1277</span> }<a name="line.1277"></a>
+<span class="sourceLineNo">1278</span><a name="line.1278"></a>
+<span class="sourceLineNo">1279</span> private void sendProcedureLoadedNotification(final long procId) {<a name="line.1279"></a>
+<span class="sourceLineNo">1280</span> if (!this.listeners.isEmpty()) {<a name="line.1280"></a>
+<span class="sourceLineNo">1281</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1281"></a>
+<span class="sourceLineNo">1282</span> try {<a name="line.1282"></a>
+<span class="sourceLineNo">1283</span> listener.procedureLoaded(procId);<a name="line.1283"></a>
+<span class="sourceLineNo">1284</span> } catch (Throwable e) {<a name="line.1284"></a>
+<span class="sourceLineNo">1285</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1285"></a>
+<span class="sourceLineNo">1286</span> }<a name="line.1286"></a>
+<span class="sourceLineNo">1287</span> }<a name="line.1287"></a>
+<span class="sourceLineNo">1288</span> }<a name="line.1288"></a>
+<span class="sourceLineNo">1289</span> }<a name="line.1289"></a>
+<span class="sourceLineNo">1290</span><a name="line.1290"></a>
+<span class="sourceLineNo">1291</span> private void sendProcedureAddedNotification(final long procId) {<a name="line.1291"></a>
+<span class="sourceLineNo">1292</span> if (!this.listeners.isEmpty()) {<a name="line.1292"></a>
+<span class="sourceLineNo">1293</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1293"></a>
+<span class="sourceLineNo">1294</span> try {<a name="line.1294"></a>
+<span class="sourceLineNo">1295</span> listener.procedureAdded(procId);<a name="line.1295"></a>
+<span class="sourceLineNo">1296</span> } catch (Throwable e) {<a name="line.1296"></a>
+<span class="sourceLineNo">1297</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1297"></a>
+<span class="sourceLineNo">1298</span> }<a name="line.1298"></a>
+<span class="sourceLineNo">1299</span> }<a name="line.1299"></a>
+<span class="sourceLineNo">1300</span> }<a name="line.1300"></a>
+<span class="sourceLineNo">1301</span> }<a name="line.1301"></a>
+<span class="sourceLineNo">1302</span><a name="line.1302"></a>
+<span class="sourceLineNo">1303</span> private void sendProcedureFinishedNotification(final long procId) {<a name="line.1303"></a>
+<span class="sourceLineNo">1304</span> if (!this.listeners.isEmpty()) {<a name="line.1304"></a>
+<span class="sourceLineNo">1305</span> for (ProcedureExecutorListener listener: this.listeners) {<a name="line.1305"></a>
+<span class="sourceLineNo">1306</span> try {<a name="line.1306"></a>
+<span class="sourceLineNo">1307</span> listener.procedureFinished(procId);<a name="line.1307"></a>
+<span class="sourceLineNo">1308</span> } catch (Throwable e) {<a name="line.1308"></a>
+<span class="sourceLineNo">1309</span> LOG.error("Listener " + listener + " had an error: " + e.getMessage(), e);<a name="line.1309"></a>
+<span class="sourceLineNo">1310</span> }<a name="line.1310"></a>
+<span class="sourceLineNo">1311</span> }<a name="line.1311"></a>
+<span class="sourceLineNo">1312</span> }<a name="line.1312"></a>
+<span class="sourceLineNo">1313</span> }<a name="line.1313"></a>
+<span class="sourceLineNo">1314</span><a name="line.1314"></a>
+<span class="sourceLineNo">1315</span> // ==========================================================================<a name="line.1315"></a>
+<span class="sourceLineNo">1316</span> // Procedure IDs helpers<a name="line.1316"></a>
+<span class="sourceLineNo">1317</span> // ==========================================================================<a name="line.1317"></a>
+<span class="sourceLineNo">1318</span> private long nextProcId() {<a name="line.1318"></a>
+<span class="sourceLineNo">1319</span> long procId = lastProcId.incrementAndGet();<a name="line.1319"></a>
+<span class="sourceLineNo">1320</span> if (procId < 0) {<a name="line.1320"></a>
+<span class="sourceLineNo">1321</span> while (!lastProcId.compareAndSet(procId, 0)) {<a name="line.1321"></a>
+<span class="sourceLineNo">1322</span> procId = lastProcId.get();<a name="line.1322"></a>
+<span class="sourceLineNo">1323</span> if (procId >= 0) {<a name="line.1323"></a>
+<span class="sourceLineNo">1324</span> break;<a name="line.1324"></a>
+<span class="sourceLineNo">1325</span> }<a name="line.1325"></a>
+<span class="sourceLineNo">1326</span> }<a name="line.1326"></a>
+<span class="sourceLineNo">1327</span> while (procedures.containsKey(procId)) {<a name="line.1327"></a>
+<span class="sourceLineNo">1328</span> procId = lastProcId.incrementAndGet();<a name="line.1328"></a>
+<span class="sourceLineNo">1329</span> }<a name="line.1329"></a>
+<span class="sourceLineNo">1330</span> }<a name="line.1330"></a>
+<span class="sourceLineNo">1331</span> assert procId >= 0 : "Invalid procId " + procId;<a name="line.1331"></a>
+<span class="sourceLineNo">1332</span> return procId;<a name="line.1332"></a>
+<span class="sourceLineNo">1333</span> }<a name="line.1333"></a>
+<span class="sourceLineNo">1334</span><a name="line.1334"></a>
+<span class="sourceLineNo">1335</span> @VisibleForTesting<a name="line.1335"></a>
+<span class="sourceLineNo">1336</span> protected long getLastProcId() {<a name="line.1336"></a>
+<span class="sourceLineNo">1337</span> return lastProcId.get();<a name="line.1337"></a>
+<span class="sourceLineNo">1338</span> }<a name="line.1338"></a>
+<span class="sourceLineNo">1339</span><a name="line.1339"></a>
+<span class="sourceLineNo">1340</span> @VisibleForTesting<a name="line.1340"></a>
+<span class="sourceLineNo">1341</span> public Set<Long> getActiveProcIds() {<a name="line.1341"></a>
+<span class="sourceLineNo">1342</span> return procedures.keySet();<a name="line.1342"></a>
+<span class="sourceLineNo">1343</span> }<a name="line.1343"></a>
+<span class="sourceLineNo">1344</span><a name="line.1344"></a>
+<span class="sourceLineNo">1345</span> Long getRootProcedureId(Procedure<TEnvironment> proc) {<a name="line.1345"></a>
+<span class="sourceLineNo">1346</span> return Procedure.getRootProcedureId(procedures, proc);<a name="line.1346"></a>
+<span class="sourceLineNo">1347</span> }<a name="line.1347"></a>
+<span class="sourceLineNo">1348</span><a name="line.1348"></a>
+<span class="sourceLineNo">1349</span> // ==========================================================================<a name="line.1349"></a>
+<span class="sourceLineNo">1350</span> // Executions<a name="line.1350"></a>
+<span class="sourceLineNo">1351</span> // ==========================================================================<a name="line.1351"></a>
+<span class="sourceLineNo">1352</span> private void executeProcedure(Procedure<TEnvironment> proc) {<a name="line.1352"></a>
+<span class="sourceLineNo">1353</span> if (proc.isFinished()) {<a name="line.1353"></a>
+<span class="sourceLineNo">1354</span> LOG.debug("{} is already finished, skipping execution", proc);<a name="line.1354"></a>
<span class="sourceLineNo">1355</span> return;<a name="line.1355"></a>
<span class="sourceLineNo">1356</span> }<a name="line.1356"></a>
-<span class="sourceLineNo">1357</span><a name="line.1357"></a>
-<span class="sourceLineNo">1358</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.1358"></a>
-<span class="sourceLineNo">1359</span> if (procStack == null) {<a name="line.1359"></a>
-<span class="sourceLineNo">1360</span> LOG.warn("RootProcedureState is null for " + proc.getProcId());<a name="line.1360"></a>
-<span class="sourceLineNo">1361</span> return;<a name="line.1361"></a>
-<span class="sourceLineNo">1362</span> }<a name="line.1362"></a>
-<span class="sourceLineNo">1363</span> do {<a name="line.1363"></a>
-<span class="sourceLineNo">1364</span> // Try to acquire the execution<a name="line.1364"></a>
-<span class="sourceLineNo">1365</span> if (!procStack.acquire(proc)) {<a name="line.1365"></a>
-<span class="sourceLineNo">1366</span> if (procStack.setRollback()) {<a name="line.1366"></a>
-<span class="sourceLineNo">1367</span> // we have the 'rollback-lock' we can start rollingback<a name="line.1367"></a>
-<span class="sourceLineNo">1368</span> switch (executeRollback(rootProcId, procStack)) {<a name="line.1368"></a>
-<span class="sourceLineNo">1369</span> case LOCK_ACQUIRED:<a name="line.1369"></a>
-<span class="sourceLineNo">1370</span> break;<a name="line.1370"></a>
-<span class="sourceLineNo">1371</span> case LOCK_YIELD_WAIT:<a name="line.1371"></a>
-<span class="sourceLineNo">1372</span> procStack.unsetRollback();<a name="line.1372"></a>
-<span class="sourceLineNo">1373</span> scheduler.yield(proc);<a name="line.1373"></a>
-<span class="sourceLineNo">1374</span> break;<a name="line.1374"></a>
-<span class="sourceLineNo">1375</span> case LOCK_EVENT_WAIT:<a name="line.1375"></a>
-<span class="sourceLineNo">1376</span> LOG.info("LOCK_EVENT_WAIT rollback..." + proc);<a name="line.1376"></a>
-<span class="sourceLineNo">1377</span> procStack.unsetRollback();<a name="line.1377"></a>
-<span class="sourceLineNo">1378</span> break;<a name="line.1378"></a>
-<span class="sourceLineNo">1379</span> default:<a name="line.1379"></a>
-<span class="sourceLineNo">1380</span> throw new UnsupportedOperationException();<a name="line.1380"></a>
-<span class="sourceLineNo">1381</span> }<a name="line.1381"></a>
-<span class="sourceLineNo">1382</span> } else {<a name="line.1382"></a>
-<span class="sourceLineNo">1383</span> // if we can't rollback means that some child is still running.<a name="line.1383"></a>
-<span class="sourceLineNo">1384</span> // the rollback will be executed after all the children are done.<a name="line.1384"></a>
-<span class="sourceLineNo">1385</span> // If the procedure was never executed, remove and mark it as rolledback.<a name="line.1385"></a>
-<span class="sourceLineNo">1386</span> if (!proc.wasExecuted()) {<a name="line.1386"></a>
-<span class="sourceLineNo">1387</span> switch (executeRollback(proc)) {<a name="line.1387"></a>
-<span class="sourceLineNo">1388</span> case LOCK_ACQUIRED:<a name="line.1388"></a>
-<span class="sourceLineNo">1389</span> break;<a name="line.1389"></a>
-<span class="sourceLineNo">1390</span> case LOCK_YIELD_WAIT:<a name="line.1390"></a>
-<span class="sourceLineNo">1391</span> scheduler.yield(proc);<a name="line.1391"></a>
-<span class="sourceLineNo">1392</span> break;<a name="line.1392"></a>
-<span class="sourceLineNo">1393</span> case LOCK_EVENT_WAIT:<a name="line.1393"></a>
-<span class="sourceLineNo">1394</span> LOG.info("LOCK_EVENT_WAIT can't rollback child running?..." + proc);<a name="line.1394"></a>
-<span class="sourceLineNo">1395</span> break;<a name="line.1395"></a>
-<span class="sourceLineNo">1396</span> default:<a name="line.1396"></a>
-<span class="sourceLineNo">1397</span> throw new UnsupportedOperationException();<a name="line.1397"></a>
-<span class="sourceLineNo">1398</span> }<a name="line.1398"></a>
-<span class="sourceLineNo">1399</span> }<a name="line.1399"></a>
-<span class="sourceLineNo">1400</span> }<a name="line.1400"></a>
-<span class="sourceLineNo">1401</span> break;<a name="line.1401"></a>
-<span class="sourceLineNo">1402</span> }<a name="line.1402"></a>
-<span class="sourceLineNo">1403</span><a name="line.1403"></a>
-<span class="sourceLineNo">1404</span> // Execute the procedure<a name="line.1404"></a>
-<span class="sourceLineNo">1405</span> assert proc.getState() == ProcedureState.RUNNABLE : proc;<a name="line.1405"></a>
-<span class="sourceLineNo">1406</span> // Note that lock is NOT about concurrency but rather about ensuring<a name="line.1406"></a>
-<span class="sourceLineNo">1407</span> // ownership of a procedure of an entity such as a region or table<a name="line.1407"></a>
-<span class="sourceLineNo">1408</span> LockState lockState = acquireLock(proc);<a name="line.1408"></a>
-<span class="sourceLineNo">1409</span> switch (lockState) {<a name="line.1409"></a>
-<span class="sourceLineNo">1410</span> case LOCK_ACQUIRED:<a name="line.1410"></a>
-<span class="sourceLineNo">1411</span> execProcedure(procStack, proc);<a name="line.1411"></a>
-<span class="sourceLineNo">1412</span> break;<a name="line.1412"></a>
-<span class="sourceLineNo">1413</span> case LOCK_YIELD_WAIT:<a name="line.1413"></a>
-<span class="sourceLineNo">1414</span> LOG.info(lockState + " " + proc);<a name="line.1414"></a>
-<span class="sourceLineNo">1415</span> scheduler.yield(proc);<a name="line.1415"></a>
-<span class="sourceLineNo">1416</span> break;<a name="line.1416"></a>
-<span class="sourceLineNo">1417</span> case LOCK_EVENT_WAIT:<a name="line.1417"></a>
-<span class="sourceLineNo">1418</span> // Someone will wake us up when the lock is available<a name="line.1418"></a>
-<span class="sourceLineNo">1419</span> LOG.debug(lockState + " " + proc);<a name="line.1419"></a>
-<span class="sourceLineNo">1420</span> break;<a name="line.1420"></a>
-<span class="sourceLineNo">1421</span> default:<a name="line.1421"></a>
-<span class="sourceLineNo">1422</span> throw new UnsupportedOperationException();<a name="line.1422"></a>
-<span class="sourceLineNo">1423</span> }<a name="line.1423"></a>
-<span class="sourceLineNo">1424</span> procStack.release(proc);<a name="line.1424"></a>
-<span class="sourceLineNo">1425</span><a name="line.1425"></a>
-<span class="sourceLineNo">1426</span> if (proc.isSuccess()) {<a name="line.1426"></a>
-<span class="sourceLineNo">1427</span> // update metrics on finishing the procedure<a name="line.1427"></a>
-<span class="sourceLineNo">1428</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);<a name="line.1428"></a>
-<span class="sourceLineNo">1429</span> LOG.info("Finished " + proc + " in " + StringUtils.humanTimeDiff(proc.elapsedTime()));<a name="line.1429"></a>
-<span class="sourceLineNo">1430</span> // Finalize the procedure state<a name="line.1430"></a>
-<span class="sourceLineNo">1431</span> if (proc.getProcId() == rootProcId) {<a name="line.1431"></a>
-<span class="sourceLineNo">1432</span> procedureFinished(proc);<a name="line.1432"></a>
-<span class="sourceLineNo">1433</span> } else {<a name="line.1433"></a>
-<span class="sourceLineNo">1434</span> execCompletionCleanup(proc);<a name="line.1434"></a>
-<span class="sourceLineNo">1435</span> }<a name="line.1435"></a>
-<span class="sourceLineNo">1436</span> break;<a name="line.1436"></a>
-<span class="sourceLineNo">1437</span> }<a name="line.1437"></a>
-<span class="sourceLineNo">1438</span> } while (procStack.isFailed());<a name="line.1438"></a>
-<span class="sourceLineNo">1439</span> }<a name="line.1439"></a>
-<span class="sourceLineNo">1440</span><a name="line.1440"></a>
-<span class="sourceLineNo">1441</span> private LockState acquireLock(Procedure<TEnvironment> proc) {<a name="line.1441"></a>
-<span class="sourceLineNo">1442</span> TEnvironment env = getEnvironment();<a name="line.1442"></a>
-<span class="sourceLineNo">1443</span> // if holdLock is true, then maybe we already have the lock, so just return LOCK_ACQUIRED if<a name="line.1443"></a>
-<span class="sourceLineNo">1444</span> // hasLock is true.<a name="line.1444"></a>
-<span class="sourceLineNo">1445</span> if (proc.hasLock()) {<a name="line.1445"></a>
-<span class="sourceLineNo">1446</span> return LockState.LOCK_ACQUIRED;<a name="line.1446"></a>
-<span class="sourceLineNo">1447</span> }<a name="line.1447"></a>
-<span class="sourceLineNo">1448</span> return proc.doAcquireLock(env, store);<a name="line.1448"></a>
-<span class="sourceLineNo">1449</span> }<a name="line.1449"></a>
-<span class="sourceLineNo">1450</span><a name="line.1450"></a>
-<span class="sourceLineNo">1451</span> private void releaseLock(Procedure<TEnvironment> proc, boolean force) {<a name="line.1451"></a>
-<span class="sourceLineNo">1452</span> TEnvironment env = getEnvironment();<a name="line.1452"></a>
-<span class="sourceLineNo">1453</span> // For how the framework works, we know that we will always have the lock<a name="line.1453"></a>
-<span class="sourceLineNo">1454</span> // when we call releaseLock(), so we can avoid calling proc.hasLock()<a name="line.1454"></a>
-<span class="sourceLineNo">1455</span> if (force || !proc.holdLock(env) || proc.isFinished()) {<a name="line.1455"></a>
-<span class="sourceLineNo">1456</span> proc.doReleaseLock(env, store);<a name="line.1456"></a>
-<span class="sourceLineNo">1457</span> }<a name="line.1457"></a>
-<span class="sourceLineNo">1458</span> }<a name="line.1458"></a>
-<span class="sourceLineNo">1459</span><a name="line.1459"></a>
-<span class="sourceLineNo">1460</span> /**<a name="line.1460"></a>
-<span class="sourceLineNo">1461</span> * Execute the rollback of the full procedure stack. Once the procedure is rolledback, the<a name="line.1461"></a>
-<span class="sourceLineNo">1462</span> * root-procedure will be visible as finished to user, and the result will be the fatal exception.<a name="line.1462"></a>
-<span class="sourceLineNo">1463</span> */<a name="line.1463"></a>
-<span class="sourceLineNo">1464</span> private LockState executeRollback(long rootProcId, RootProcedureState<TEnvironment> procStack) {<a name="line.1464"></a>
-<span class="sourceLineNo">1465</span> Procedure<TEnvironment> rootProc = procedures.get(rootProcId);<a name="line.1465"></a>
-<span class="sourceLineNo">1466</span> RemoteProcedureException exception = rootProc.getException();<a name="line.1466"></a>
-<span class="sourceLineNo">1467</span> // TODO: This needs doc. The root proc doesn't have an exception. Maybe we are<a name="line.1467"></a>
-<span class="sourceLineNo">1468</span> // rolling back because the subprocedure does. Clarify.<a name="line.1468"></a>
-<span class="sourceLineNo">1469</span> if (exception == null) {<a name="line.1469"></a>
-<span class="sourceLineNo">1470</span> exception = procStack.getException();<a name="line.1470"></a>
-<span class="sourceLineNo">1471</span> rootProc.setFailure(exception);<a name="line.1471"></a>
-<span class="sourceLineNo">1472</span> store.update(rootProc);<a name="line.1472"></a>
-<span class="sourceLineNo">1473</span> }<a name="line.1473"></a>
-<span class="sourceLineNo">1474</span><a name="line.1474"></a>
-<span class="sourceLineNo">1475</span> List<Procedure<TEnvironment>> subprocStack = procStack.getSubproceduresStack();<a name="line.1475"></a>
-<span class="sourceLineNo">1476</span> assert subprocStack != null : "Called rollback with no steps executed rootProc=" + rootProc;<a name="line.1476"></a>
-<span class="sourceLineNo">1477</span><a name="line.1477"></a>
-<span class="sourceLineNo">1478</span> int stackTail = subprocStack.size();<a name="line.1478"></a>
-<span class="sourceLineNo">1479</span> while (stackTail-- > 0) {<a name="line.1479"></a>
-<span class="sourceLineNo">1480</span> Procedure<TEnvironment> proc = subprocStack.get(stackTail);<a name="line.1480"></a>
-<span class="sourceLineNo">1481</span> IdLock.Entry lockEntry = null;<a name="line.1481"></a>
-<span class="sourceLineNo">1482</span> // Hold the execution lock if it is not held by us. The IdLock is not reentrant so we need<a name="line.1482"></a>
-<span class="sourceLineNo">1483</span> // this check, as the worker will hold the lock before executing a procedure. This is the only<a name="line.1483"></a>
-<span class="sourceLineNo">1484</span> // place where we may hold two procedure execution locks, and there is a fence in the<a name="line.1484"></a>
-<span class="sourceLineNo">1485</span> // RootProcedureState where we can make sure that only one worker can execute the rollback of<a name="line.1485"></a>
-<span class="sourceLineNo">1486</span> // a RootProcedureState, so there is no dead lock problem. And the lock here is necessary to<a name="line.1486"></a>
-<span class="sourceLineNo">1487</span> // prevent race between us and the force update thread.<a name="line.1487"></a>
-<span class="sourceLineNo">1488</span> if (!procExecutionLock.isHeldByCurrentThread(proc.getProcId())) {<a name="line.1488"></a>
-<span class="sourceLineNo">1489</span> try {<a name="line.1489"></a>
-<span class="sourceLineNo">1490</span> lockEntry = procExecutionLock.getLockEntry(proc.getProcId());<a name="line.1490"></a>
-<span class="sourceLineNo">1491</span> } catch (IOException e) {<a name="line.1491"></a>
-<span class="sourceLineNo">1492</span> // can only happen if interrupted, so not a big deal to propagate it<a name="line.1492"></a>
-<span class="sourceLineNo">1493</span> throw new UncheckedIOException(e);<a name="line.1493"></a>
-<span class="sourceLineNo">1494</span> }<a name="line.1494"></a>
-<span class="sourceLineNo">1495</span> }<a name="line.1495"></a>
-<span class="sourceLineNo">1496</span> try {<a name="line.1496"></a>
-<span class="sourceLineNo">1497</span> // For the sub procedures which are successfully finished, we do not rollback them.<a name="line.1497"></a>
-<span class="sourceLineNo">1498</span> // Typically, if we want to rollback a procedure, we first need to rollback it, and then<a name="line.1498"></a>
-<span class="sourceLineNo">1499</span> // recursively rollback its ancestors. The state changes which are done by sub procedures<a name="line.1499"></a>
-<span class="sourceLineNo">1500</span> // should be handled by parent procedures when rolling back. For example, when rolling back<a name="line.1500"></a>
-<span class="sourceLineNo">1501</span> // a MergeTableProcedure, we will schedule new procedures to bring the offline regions<a name="line.1501"></a>
-<span class="sourceLineNo">1502</span> // online, instead of rolling back the original procedures which offlined the regions(in<a name="line.1502"></a>
-<span class="sourceLineNo">1503</span> // fact these procedures can not be rolled back...).<a name="line.1503"></a>
-<span class="sourceLineNo">1504</span> if (proc.isSuccess()) {<a name="line.1504"></a>
-<span class="sourceLineNo">1505</span> // Just do the cleanup work, without actually executing the rollback<a name="line.1505"></a>
-<span class="sourceLineNo">1506</span> subprocStack.remove(stackTail);<a name="line.1506"></a>
-<span class="sourceLineNo">1507</span> cleanupAfterRollbackOneStep(proc);<a name="line.1507"></a>
-<span class="sourceLineNo">1508</span> continue;<a name="line.1508"></a>
-<span class="sourceLineNo">1509</span> }<a name="line.1509"></a>
-<span class="sourceLineNo">1510</span> LockState lockState = acquireLock(proc);<a name="line.1510"></a>
-<span class="sourceLineNo">1511</span> if (lockState != LockState.LOCK_ACQUIRED) {<a name="line.1511"></a>
-<span class="sourceLineNo">1512</span> // can't take a lock on the procedure, add the root-proc back on the<a name="line.1512"></a>
-<span class="sourceLineNo">1513</span> // queue waiting for the lock availability<a name="line.1513"></a>
-<span class="sourceLineNo">1514</span> return lockState;<a name="line.1514"></a>
-<span class="sourceLineNo">1515</span> }<a name="line.1515"></a>
-<span class="sourceLineNo">1516</span><a name="line.1516"></a>
-<span class="sourceLineNo">1517</span> lockState = executeRollback(proc);<a name="line.1517"></a>
-<span class="sourceLineNo">1518</span> releaseLock(proc, false);<a name="line.1518"></a>
-<span class="sourceLineNo">1519</span> boolean abortRollback = lockState != LockState.LOCK_ACQUIRED;<a name="line.1519"></a>
-<span class="sourceLineNo">1520</span> abortRollback |= !isRunning() || !store.isRunning();<a name="line.1520"></a>
-<span class="sourceLineNo">1521</span><a name="line.1521"></a>
-<span class="sourceLineNo">1522</span> // allows to kill the executor before something is stored to the wal.<a name="line.1522"></a>
-<span class="sourceLineNo">1523</span> // useful to test the procedure recovery.<a name="line.1523"></a>
-<span class="sourceLineNo">1524</span> if (abortRollback) {<a name="line.1524"></a>
-<span class="sourceLineNo">1525</span> return lockState;<a name="line.1525"></a>
-<span class="sourceLineNo">1526</span> }<a name="line.1526"></a>
-<span class="sourceLineNo">1527</span><a name="line.1527"></a>
-<span class="sourceLineNo">1528</span> subprocStack.remove(stackTail);<a name="line.1528"></a>
-<span class="sourceLineNo">1529</span><a name="line.1529"></a>
-<span class="sourceLineNo">1530</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1530"></a>
-<span class="sourceLineNo">1531</span> // if the proc is already finished, do not yield<a name="line.1531"></a>
-<span class="sourceLineNo">1532</span> if (!proc.isFinished() && proc.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1532"></a>
-<span class="sourceLineNo">1533</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1533"></a>
-<span class="sourceLineNo">1534</span> }<a name="line.1534"></a>
-<span class="sourceLineNo">1535</span><a name="line.1535"></a>
-<span class="sourceLineNo">1536</span> if (proc != rootProc) {<a name="line.1536"></a>
-<span class="sourceLineNo">1537</span> execCompletionCleanup(proc);<a name="line.1537"></a>
-<span class="sourceLineNo">1538</span> }<a name="line.1538"></a>
-<span class="sourceLineNo">1539</span> } finally {<a name="line.1539"></a>
-<span class="sourceLineNo">1540</span> if (lockEntry != null) {<a name="line.1540"></a>
-<span class="sourceLineNo">1541</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1541"></a>
-<span class="sourceLineNo">1542</span> }<a name="line.1542"></a>
-<span class="sourceLineNo">1543</span> }<a name="line.1543"></a>
-<span class="sourceLineNo">1544</span> }<a name="line.1544"></a>
-<span class="sourceLineNo">1545</span><a name="line.1545"></a>
-<span class="sourceLineNo">1546</span> // Finalize the procedure state<a name="line.1546"></a>
-<span class="sourceLineNo">1547</span> LOG.info("Rolled back {} exec-time={}", rootProc,<a name="line.1547"></a>
-<span class="sourceLineNo">1548</span> StringUtils.humanTimeDiff(rootProc.elapsedTime()));<a name="line.1548"></a>
-<span class="sourceLineNo">1549</span> procedureFinished(rootProc);<a name="line.1549"></a>
-<span class="sourceLineNo">1550</span> return LockState.LOCK_ACQUIRED;<a name="line.1550"></a>
-<span class="sourceLineNo">1551</span> }<a name="line.1551"></a>
+<span class="sourceLineNo">1357</span> final Long rootProcId = getRootProcedureId(proc);<a name="line.1357"></a>
+<span class="sourceLineNo">1358</span> if (rootProcId == null) {<a name="line.1358"></a>
+<span class="sourceLineNo">1359</span> // The 'proc' was ready to run but the root procedure was rolledback<a name="line.1359"></a>
+<span class="sourceLineNo">1360</span> LOG.warn("Rollback because parent is done/rolledback proc=" + proc);<a name="line.1360"></a>
+<span class="sourceLineNo">1361</span> executeRollback(proc);<a name="line.1361"></a>
+<span class="sourceLineNo">1362</span> return;<a name="line.1362"></a>
+<span class="sourceLineNo">1363</span> }<a name="line.1363"></a>
+<span class="sourceLineNo">1364</span><a name="line.1364"></a>
+<span class="sourceLineNo">1365</span> RootProcedureState<TEnvironment> procStack = rollbackStack.get(rootProcId);<a name="line.1365"></a>
+<span class="sourceLineNo">1366</span> if (procStack == null) {<a name="line.1366"></a>
+<span class="sourceLineNo">1367</span> LOG.warn("RootProcedureState is null for " + proc.getProcId());<a name="line.1367"></a>
+<span class="sourceLineNo">1368</span> return;<a name="line.1368"></a>
+<span class="sourceLineNo">1369</span> }<a name="line.1369"></a>
+<span class="sourceLineNo">1370</span> do {<a name="line.1370"></a>
+<span class="sourceLineNo">1371</span> // Try to acquire the execution<a name="line.1371"></a>
+<span class="sourceLineNo">1372</span> if (!procStack.acquire(proc)) {<a name="line.1372"></a>
+<span class="sourceLineNo">1373</span> if (procStack.setRollback()) {<a name="line.1373"></a>
+<span class="sourceLineNo">1374</span> // we have the 'rollback-lock' we can start rollingback<a name="line.1374"></a>
+<span class="sourceLineNo">1375</span> switch (executeRollback(rootProcId, procStack)) {<a name="line.1375"></a>
+<span class="sourceLineNo">1376</span> case LOCK_ACQUIRED:<a name="line.1376"></a>
+<span class="sourceLineNo">1377</span> break;<a name="line.1377"></a>
+<span class="sourceLineNo">1378</span> case LOCK_YIELD_WAIT:<a name="line.1378"></a>
+<span class="sourceLineNo">1379</span> procStack.unsetRollback();<a name="line.1379"></a>
+<span class="sourceLineNo">1380</span> scheduler.yield(proc);<a name="line.1380"></a>
+<span class="sourceLineNo">1381</span> break;<a name="line.1381"></a>
+<span class="sourceLineNo">1382</span> case LOCK_EVENT_WAIT:<a name="line.1382"></a>
+<span class="sourceLineNo">1383</span> LOG.info("LOCK_EVENT_WAIT rollback..." + proc);<a name="line.1383"></a>
+<span class="sourceLineNo">1384</span> procStack.unsetRollback();<a name="line.1384"></a>
+<span class="sourceLineNo">1385</span> break;<a name="line.1385"></a>
+<span class="sourceLineNo">1386</span> default:<a name="line.1386"></a>
+<span class="sourceLineNo">1387</span> throw new UnsupportedOperationException();<a name="line.1387"></a>
+<span class="sourceLineNo">1388</span> }<a name="line.1388"></a>
+<span class="sourceLineNo">1389</span> } else {<a name="line.1389"></a>
+<span class="sourceLineNo">1390</span> // if we can't rollback means that some child is still running.<a name="line.1390"></a>
+<span class="sourceLineNo">1391</span> // the rollback will be executed after all the children are done.<a name="line.1391"></a>
+<span class="sourceLineNo">1392</span> // If the procedure was never executed, remove and mark it as rolledback.<a name="line.1392"></a>
+<span class="sourceLineNo">1393</span> if (!proc.wasExecuted()) {<a name="line.1393"></a>
+<span class="sourceLineNo">1394</span> switch (executeRollback(proc)) {<a name="line.1394"></a>
+<span class="sourceLineNo">1395</span> case LOCK_ACQUIRED:<a name="line.1395"></a>
+<span class="sourceLineNo">1396</span> break;<a name="line.1396"></a>
+<span class="sourceLineNo">1397</span> case LOCK_YIELD_WAIT:<a name="line.1397"></a>
+<span class="sourceLineNo">1398</span> scheduler.yield(proc);<a name="line.1398"></a>
+<span class="sourceLineNo">1399</span> break;<a name="line.1399"></a>
+<span class="sourceLineNo">1400</span> case LOCK_EVENT_WAIT:<a name="line.1400"></a>
+<span class="sourceLineNo">1401</span> LOG.info("LOCK_EVENT_WAIT can't rollback child running?..." + proc);<a name="line.1401"></a>
+<span class="sourceLineNo">1402</span> break;<a name="line.1402"></a>
+<span class="sourceLineNo">1403</span> default:<a name="line.1403"></a>
+<span class="sourceLineNo">1404</span> throw new UnsupportedOperationException();<a name="line.1404"></a>
+<span class="sourceLineNo">1405</span> }<a name="line.1405"></a>
+<span class="sourceLineNo">1406</span> }<a name="line.1406"></a>
+<span class="sourceLineNo">1407</span> }<a name="line.1407"></a>
+<span class="sourceLineNo">1408</span> break;<a name="line.1408"></a>
+<span class="sourceLineNo">1409</span> }<a name="line.1409"></a>
+<span class="sourceLineNo">1410</span><a name="line.1410"></a>
+<span class="sourceLineNo">1411</span> // Execute the procedure<a name="line.1411"></a>
+<span class="sourceLineNo">1412</span> assert proc.getState() == ProcedureState.RUNNABLE : proc;<a name="line.1412"></a>
+<span class="sourceLineNo">1413</span> // Note that lock is NOT about concurrency but rather about ensuring<a name="line.1413"></a>
+<span class="sourceLineNo">1414</span> // ownership of a procedure of an entity such as a region or table<a name="line.1414"></a>
+<span class="sourceLineNo">1415</span> LockState lockState = acquireLock(proc);<a name="line.1415"></a>
+<span class="sourceLineNo">1416</span> switch (lockState) {<a name="line.1416"></a>
+<span class="sourceLineNo">1417</span> case LOCK_ACQUIRED:<a name="line.1417"></a>
+<span class="sourceLineNo">1418</span> execProcedure(procStack, proc);<a name="line.1418"></a>
+<span class="sourceLineNo">1419</span> break;<a name="line.1419"></a>
+<span class="sourceLineNo">1420</span> case LOCK_YIELD_WAIT:<a name="line.1420"></a>
+<span class="sourceLineNo">1421</span> LOG.info(lockState + " " + proc);<a name="line.1421"></a>
+<span class="sourceLineNo">1422</span> scheduler.yield(proc);<a name="line.1422"></a>
+<span class="sourceLineNo">1423</span> break;<a name="line.1423"></a>
+<span class="sourceLineNo">1424</span> case LOCK_EVENT_WAIT:<a name="line.1424"></a>
+<span class="sourceLineNo">1425</span> // Someone will wake us up when the lock is available<a name="line.1425"></a>
+<span class="sourceLineNo">1426</span> LOG.debug(lockState + " " + proc);<a name="line.1426"></a>
+<span class="sourceLineNo">1427</span> break;<a name="line.1427"></a>
+<span class="sourceLineNo">1428</span> default:<a name="line.1428"></a>
+<span class="sourceLineNo">1429</span> throw new UnsupportedOperationException();<a name="line.1429"></a>
+<span class="sourceLineNo">1430</span> }<a name="line.1430"></a>
+<span class="sourceLineNo">1431</span> procStack.release(proc);<a name="line.1431"></a>
+<span class="sourceLineNo">1432</span><a name="line.1432"></a>
+<span class="sourceLineNo">1433</span> if (proc.isSuccess()) {<a name="line.1433"></a>
+<span class="sourceLineNo">1434</span> // update metrics on finishing the procedure<a name="line.1434"></a>
+<span class="sourceLineNo">1435</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);<a name="line.1435"></a>
+<span class="sourceLineNo">1436</span> LOG.info("Finished " + proc + " in " + StringUtils.humanTimeDiff(proc.elapsedTime()));<a name="line.1436"></a>
+<span class="sourceLineNo">1437</span> // Finalize the procedure state<a name="line.1437"></a>
+<span class="sourceLineNo">1438</span> if (proc.getProcId() == rootProcId) {<a name="line.1438"></a>
+<span class="sourceLineNo">1439</span> procedureFinished(proc);<a name="line.1439"></a>
+<span class="sourceLineNo">1440</span> } else {<a name="line.1440"></a>
+<span class="sourceLineNo">1441</span> execCompletionCleanup(proc);<a name="line.1441"></a>
+<span class="sourceLineNo">1442</span> }<a name="line.1442"></a>
+<span class="sourceLineNo">1443</span> break;<a name="line.1443"></a>
+<span class="sourceLineNo">1444</span> }<a name="line.1444"></a>
+<span class="sourceLineNo">1445</span> } while (procStack.isFailed());<a name="line.1445"></a>
+<span class="sourceLineNo">1446</span> }<a name="line.1446"></a>
+<span class="sourceLineNo">1447</span><a name="line.1447"></a>
+<span class="sourceLineNo">1448</span> private LockState acquireLock(Procedure<TEnvironment> proc) {<a name="line.1448"></a>
+<span class="sourceLineNo">1449</span> TEnvironment env = getEnvironment();<a name="line.1449"></a>
+<span class="sourceLineNo">1450</span> // if holdLock is true, then maybe we already have the lock, so just return LOCK_ACQUIRED if<a name="line.1450"></a>
+<span class="sourceLineNo">1451</span> // hasLock is true.<a name="line.1451"></a>
+<span class="sourceLineNo">1452</span> if (proc.hasLock()) {<a name="line.1452"></a>
+<span class="sourceLineNo">1453</span> return LockState.LOCK_ACQUIRED;<a name="line.1453"></a>
+<span class="sourceLineNo">1454</span> }<a name="line.1454"></a>
+<span class="sourceLineNo">1455</span> return proc.doAcquireLock(env, store);<a name="line.1455"></a>
+<span class="sourceLineNo">1456</span> }<a name="line.1456"></a>
+<span class="sourceLineNo">1457</span><a name="line.1457"></a>
+<span class="sourceLineNo">1458</span> private void releaseLock(Procedure<TEnvironment> proc, boolean force) {<a name="line.1458"></a>
+<span class="sourceLineNo">1459</span> TEnvironment env = getEnvironment();<a name="line.1459"></a>
+<span class="sourceLineNo">1460</span> // For how the framework works, we know that we will always have the lock<a name="line.1460"></a>
+<span class="sourceLineNo">1461</span> // when we call releaseLock(), so we can avoid calling proc.hasLock()<a name="line.1461"></a>
+<span class="sourceLineNo">1462</span> if (force || !proc.holdLock(env) || proc.isFinished()) {<a name="line.1462"></a>
+<span class="sourceLineNo">1463</span> proc.doReleaseLock(env, store);<a name="line.1463"></a>
+<span class="sourceLineNo">1464</span> }<a name="line.1464"></a>
+<span class="sourceLineNo">1465</span> }<a name="line.1465"></a>
+<span class="sourceLineNo">1466</span><a name="line.1466"></a>
+<span class="sourceLineNo">1467</span> /**<a name="line.1467"></a>
+<span class="sourceLineNo">1468</span> * Execute the rollback of the full procedure stack. Once the procedure is rolledback, the<a name="line.1468"></a>
+<span class="sourceLineNo">1469</span> * root-procedure will be visible as finished to user, and the result will be the fatal exception.<a name="line.1469"></a>
+<span class="sourceLineNo">1470</span> */<a name="line.1470"></a>
+<span class="sourceLineNo">1471</span> private LockState executeRollback(long rootProcId, RootProcedureState<TEnvironment> procStack) {<a name="line.1471"></a>
+<span class="sourceLineNo">1472</span> Procedure<TEnvironment> rootProc = procedures.get(rootProcId);<a name="line.1472"></a>
+<span class="sourceLineNo">1473</span> RemoteProcedureException exception = rootProc.getException();<a name="line.1473"></a>
+<span class="sourceLineNo">1474</span> // TODO: This needs doc. The root proc doesn't have an exception. Maybe we are<a name="line.1474"></a>
+<span class="sourceLineNo">1475</span> // rolling back because the subprocedure does. Clarify.<a name="line.1475"></a>
+<span class="sourceLineNo">1476</span> if (exception == null) {<a name="line.1476"></a>
+<span class="sourceLineNo">1477</span> exception = procStack.getException();<a name="line.1477"></a>
+<span class="sourceLineNo">1478</span> rootProc.setFailure(exception);<a name="line.1478"></a>
+<span class="sourceLineNo">1479</span> store.update(rootProc);<a name="line.1479"></a>
+<span class="sourceLineNo">1480</span> }<a name="line.1480"></a>
+<span class="sourceLineNo">1481</span><a name="line.1481"></a>
+<span class="sourceLineNo">1482</span> List<Procedure<TEnvironment>> subprocStack = procStack.getSubproceduresStack();<a name="line.1482"></a>
+<span class="sourceLineNo">1483</span> assert subprocStack != null : "Called rollback with no steps executed rootProc=" + rootProc;<a name="line.1483"></a>
+<span class="sourceLineNo">1484</span><a name="line.1484"></a>
+<span class="sourceLineNo">1485</span> int stackTail = subprocStack.size();<a name="line.1485"></a>
+<span class="sourceLineNo">1486</span> while (stackTail-- > 0) {<a name="line.1486"></a>
+<span class="sourceLineNo">1487</span> Procedure<TEnvironment> proc = subprocStack.get(stackTail);<a name="line.1487"></a>
+<span class="sourceLineNo">1488</span> IdLock.Entry lockEntry = null;<a name="line.1488"></a>
+<span class="sourceLineNo">1489</span> // Hold the execution lock if it is not held by us. The IdLock is not reentrant so we need<a name="line.1489"></a>
+<span class="sourceLineNo">1490</span> // this check, as the worker will hold the lock before executing a procedure. This is the only<a name="line.1490"></a>
+<span class="sourceLineNo">1491</span> // place where we may hold two procedure execution locks, and there is a fence in the<a name="line.1491"></a>
+<span class="sourceLineNo">1492</span> // RootProcedureState where we can make sure that only one worker can execute the rollback of<a name="line.1492"></a>
+<span class="sourceLineNo">1493</span> // a RootProcedureState, so there is no dead lock problem. And the lock here is necessary to<a name="line.1493"></a>
+<span class="sourceLineNo">1494</span> // prevent race between us and the force update thread.<a name="line.1494"></a>
+<span class="sourceLineNo">1495</span> if (!procExecutionLock.isHeldByCurrentThread(proc.getProcId())) {<a name="line.1495"></a>
+<span class="sourceLineNo">1496</span> try {<a name="line.1496"></a>
+<span class="sourceLineNo">1497</span> lockEntry = procExecutionLock.getLockEntry(proc.getProcId());<a name="line.1497"></a>
+<span class="sourceLineNo">1498</span> } catch (IOException e) {<a name="line.1498"></a>
+<span class="sourceLineNo">1499</span> // can only happen if interrupted, so not a big deal to propagate it<a name="line.1499"></a>
+<span class="sourceLineNo">1500</span> throw new UncheckedIOException(e);<a name="line.1500"></a>
+<span class="sourceLineNo">1501</span> }<a name="line.1501"></a>
+<span class="sourceLineNo">1502</span> }<a name="line.1502"></a>
+<span class="sourceLineNo">1503</span> try {<a name="line.1503"></a>
+<span class="sourceLineNo">1504</span> // For the sub procedures which are successfully finished, we do not rollback them.<a name="line.1504"></a>
+<span class="sourceLineNo">1505</span> // Typically, if we want to rollback a procedure, we first need to rollback it, and then<a name="line.1505"></a>
+<span class="sourceLineNo">1506</span> // recursively rollback its ancestors. The state changes which are done by sub procedures<a name="line.1506"></a>
+<span class="sourceLineNo">1507</span> // should be handled by parent procedures when rolling back. For example, when rolling back<a name="line.1507"></a>
+<span class="sourceLineNo">1508</span> // a MergeTableProcedure, we will schedule new procedures to bring the offline regions<a name="line.1508"></a>
+<span class="sourceLineNo">1509</span> // online, instead of rolling back the original procedures which offlined the regions(in<a name="line.1509"></a>
+<span class="sourceLineNo">1510</span> // fact these procedures can not be rolled back...).<a name="line.1510"></a>
+<span class="sourceLineNo">1511</span> if (proc.isSuccess()) {<a name="line.1511"></a>
+<span class="sourceLineNo">1512</span> // Just do the cleanup work, without actually executing the rollback<a name="line.1512"></a>
+<span class="sourceLineNo">1513</span> subprocStack.remove(stackTail);<a name="line.1513"></a>
+<span class="sourceLineNo">1514</span> cleanupAfterRollbackOneStep(proc);<a name="line.1514"></a>
+<span class="sourceLineNo">1515</span> continue;<a name="line.1515"></a>
+<span class="sourceLineNo">1516</span> }<a name="line.1516"></a>
+<span class="sourceLineNo">1517</span> LockState lockState = acquireLock(proc);<a name="line.1517"></a>
+<span class="sourceLineNo">1518</span> if (lockState != LockState.LOCK_ACQUIRED) {<a name="line.1518"></a>
+<span class="sourceLineNo">1519</span> // can't take a lock on the procedure, add the root-proc back on the<a name="line.1519"></a>
+<span class="sourceLineNo">1520</span> // queue waiting for the lock availability<a name="line.1520"></a>
+<span class="sourceLineNo">1521</span> return lockState;<a name="line.1521"></a>
+<span class="sourceLineNo">1522</span> }<a name="line.1522"></a>
+<span class="sourceLineNo">1523</span><a name="line.1523"></a>
+<span class="sourceLineNo">1524</span> lockState = executeRollback(proc);<a name="line.1524"></a>
+<span class="sourceLineNo">1525</span> releaseLock(proc, false);<a name="line.1525"></a>
+<span class="sourceLineNo">1526</span> boolean abortRollback = lockState != LockState.LOCK_ACQUIRED;<a name="line.1526"></a>
+<span class="sourceLineNo">1527</span> abortRollback |= !isRunning() || !store.isRunning();<a name="line.1527"></a>
+<span class="sourceLineNo">1528</span><a name="line.1528"></a>
+<span class="sourceLineNo">1529</span> // allows to kill the executor before something is stored to the wal.<a name="line.1529"></a>
+<span class="sourceLineNo">1530</span> // useful to test the procedure recovery.<a name="line.1530"></a>
+<span class="sourceLineNo">1531</span> if (abortRollback) {<a name="line.1531"></a>
+<span class="sourceLineNo">1532</span> return lockState;<a name="line.1532"></a>
+<span class="sourceLineNo">1533</span> }<a name="line.1533"></a>
+<span class="sourceLineNo">1534</span><a name="line.1534"></a>
+<span class="sourceLineNo">1535</span> subprocStack.remove(stackTail);<a name="line.1535"></a>
+<span class="sourceLineNo">1536</span><a name="line.1536"></a>
+<span class="sourceLineNo">1537</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1537"></a>
+<span class="sourceLineNo">1538</span> // if the proc is already finished, do not yield<a name="line.1538"></a>
+<span class="sourceLineNo">1539</span> if (!proc.isFinished() && proc.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1539"></a>
+<span class="sourceLineNo">1540</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1540"></a>
+<span class="sourceLineNo">1541</span> }<a name="line.1541"></a>
+<span class="sourceLineNo">1542</span><a name="line.1542"></a>
+<span class="sourceLineNo">1543</span> if (proc != rootProc) {<a name="line.1543"></a>
+<span class="sourceLineNo">1544</span> execCompletionCleanup(proc);<a name="line.1544"></a>
+<span class="sourceLineNo">1545</span> }<a name="line.1545"></a>
+<span class="sourceLineNo">1546</span> } finally {<a name="line.1546"></a>
+<span class="sourceLineNo">1547</span> if (lockEntry != null) {<a name="line.1547"></a>
+<span class="sourceLineNo">1548</span> procExecutionLock.releaseLockEntry(lockEntry);<a name="line.1548"></a>
+<span class="sourceLineNo">1549</span> }<a name="line.1549"></a>
+<span class="sourceLineNo">1550</span> }<a name="line.1550"></a>
+<span class="sourceLineNo">1551</span> }<a name="line.1551"></a>
<span class="sourceLineNo">1552</span><a name="line.1552"></a>
-<span class="sourceLineNo">1553</span> private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) {<a name="line.1553"></a>
-<span class="sourceLineNo">1554</span> if (proc.removeStackIndex()) {<a name="line.1554"></a>
-<span class="sourceLineNo">1555</span> if (!proc.isSuccess()) {<a name="line.1555"></a>
-<span class="sourceLineNo">1556</span> proc.setState(ProcedureState.ROLLEDBACK);<a name="line.1556"></a>
-<span class="sourceLineNo">1557</span> }<a name="line.1557"></a>
-<span class="sourceLineNo">1558</span><a name="line.1558"></a>
-<span class="sourceLineNo">1559</span> // update metrics on finishing the procedure (fail)<a name="line.1559"></a>
-<span class="sourceLineNo">1560</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false);<a name="line.1560"></a>
-<span class="sourceLineNo">1561</span><a name="line.1561"></a>
-<span class="sourceLineNo">1562</span> if (proc.hasParent()) {<a name="line.1562"></a>
-<span class="sourceLineNo">1563</span> store.delete(proc.getProcId());<a name="line.1563"></a>
-<span class="sourceLineNo">1564</span> procedures.remove(proc.getProcId());<a name="line.1564"></a>
-<span class="sourceLineNo">1565</span> } else {<a name="line.1565"></a>
-<span class="sourceLineNo">1566</span> final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds();<a name="line.1566"></a>
-<span class="sourceLineNo">1567</span> if (childProcIds != null) {<a name="line.1567"></a>
-<span class="sourceLineNo">1568</span> store.delete(proc, childProcIds);<a name="line.1568"></a>
-<span class="sourceLineNo">1569</span> } else {<a name="line.1569"></a>
-<span class="sourceLineNo">1570</span> store.update(proc);<a name="line.1570"></a>
-<span class="sourceLineNo">1571</span> }<a name="line.1571"></a>
-<span class="sourceLineNo">1572</span> }<a name="line.1572"></a>
-<span class="sourceLineNo">1573</span> } else {<a name="line.1573"></a>
-<span class="sourceLineNo">1574</span> store.update(proc);<a name="line.1574"></a>
-<span class="sourceLineNo">1575</span> }<a name="line.1575"></a>
-<span class="sourceLineNo">1576</span> }<a name="line.1576"></a>
-<span class="sourceLineNo">1577</span><a name="line.1577"></a>
-<span class="sourceLineNo">1578</span> /**<a name="line.1578"></a>
-<span class="sourceLineNo">1579</span> * Execute the rollback of the procedure step.<a name="line.1579"></a>
-<span class="sourceLineNo">1580</span> * It updates the store with the new state (stack index)<a name="line.1580"></a>
-<span class="sourceLineNo">1581</span> * or will remove completly the procedure in case it is a child.<a name="line.1581"></a>
-<span class="sourceLineNo">1582</span> */<a name="line.1582"></a>
-<span class="sourceLineNo">1583</span> private LockState executeRollback(Procedure<TEnvironment> proc) {<a name="line.1583"></a>
-<span class="sourceLineNo">1584</span> try {<a name="line.1584"></a>
-<span class="sourceLineNo">1585</span> proc.doRollback(getEnvironment());<a name="line.1585"></a>
-<span class="sourceLineNo">1586</span> } catch (IOException e) {<a name="line.1586"></a>
-<span class="sourceLineNo">1587</span> LOG.debug("Roll back attempt failed for {}", proc, e);<a name="line.1587"></a>
-<span class="sourceLineNo">1588</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1588"></a>
-<span class="sourceLineNo">1589</span> } catch (InterruptedException e) {<a name="line.1589"></a>
-<span class="sourceLineNo">1590</span> handleInterruptedException(proc, e);<a name="line.1590"></a>
-<span class="sourceLineNo">1591</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1591"></a>
-<span class="sourceLineNo">1592</span> } catch (Throwable e) {<a name="line.1592"></a>
-<span class="sourceLineNo">1593</span> // Catch NullPointerExceptions or similar errors...<a name="line.1593"></a>
-<span class="sourceLineNo">1594</span> LOG.error(HBaseMarkers.FATAL, "CODE-BUG: Uncaught runtime exception for " + proc, e);<a name="line.1594"></a>
-<span class="sourceLineNo">1595</span> }<a name="line.1595"></a>
-<span class="sourceLineNo">1596</span><a name="line.1596"></a>
-<span class="sourceLineNo">1597</span> // allows to kill the executor before something is stored to the wal.<a name="line.1597"></a>
-<span class="sourceLineNo">1598</span> // useful to test the procedure recovery.<a name="line.1598"></a>
-<span class="sourceLineNo">1599</span> if (testing != null && testing.shouldKillBeforeStoreUpdate()) {<a name="line.1599"></a>
-<span class="sourceLineNo">1600</span> String msg = "TESTING: Kill before store update";<a name="line.1600"></a>
-<span class="sourceLineNo">1601</span> LOG.debug(msg);<a name="line.1601"></a>
-<span class="sourceLineNo">1602</span> stop();<a name="line.1602"></a>
-<span class="sourceLineNo">1603</span> throw new RuntimeException(msg);<a name="line.1603"></a>
-<span class="sourceLineNo">1604</span> }<a name="line.1604"></a>
-<span class="sourceLineNo">1605</span><a name="line.1605"></a>
-<span class="sourceLineNo">1606</span> cleanupAfterRollbackOneStep(proc);<a name="line.1606"></a>
-<span class="sourceLineNo">1607</span><a name="line.1607"></a>
-<span class="sourceLineNo">1608</span> return LockState.LOCK_ACQUIRED;<a name="line.1608"></a>
-<span class="sourceLineNo">1609</span> }<a name="line.1609"></a>
-<span class="sourceLineNo">1610</span><a name="line.1610"></a>
-<span class="sourceLineNo">1611</span> private void yieldProcedure(Procedure<TEnvironment> proc) {<a name="line.1611"></a>
-<span class="sourceLineNo">1612</span> releaseLock(proc, false);<a name="line.1612"></a>
-<span class="sourceLineNo">1613</span> scheduler.yield(proc);<a name="line.1613"></a>
-<span class="sourceLineNo">1614</span> }<a name="line.1614"></a>
-<span class="sourceLineNo">1615</span><a name="line.1615"></a>
-<span class="sourceLineNo">1616</span> /**<a name="line.1616"></a>
-<span class="sourceLineNo">1617</span> * Executes <code>procedure</code><a name="line.1617"></a>
-<span class="sourceLineNo">1618</span> * <ul><a name="line.1618"></a>
-<span class="sourceLineNo">1619</span> * <li>Calls the doExecute() of the procedure<a name="line.1619"></a>
-<span class="sourceLineNo">1620</span> * <li>If the procedure execution didn't fail (i.e. valid user input)<a name="line.1620"></a>
-<span class="sourceLineNo">1621</span> * <ul><a name="line.1621"></a>
-<span class="sourceLineNo">1622</span> * <li>...and returned subprocedures<a name="line.1622"></a>
-<span class="sourceLineNo">1623</span> * <ul><li>The subprocedures are initialized.<a name="line.1623"></a>
-<span class="sourceLineNo">1624</span> * <li>The subprocedures are added to the store<a name="line.1624"></a>
-<span class="sourceLineNo">1625</span> * <li>The subprocedures are added to the runnable queue<a name="line.1625"></a>
-<span class="sourceLineNo">1626</span> * <li>The procedure is now in a WAITING state, waiting for the subprocedures to complete<a name="line.1626"></a>
-<span class="sourceLineNo">1627</span> * </ul><a name="line.1627"></a>
-<span class="sourceLineNo">1628</span> * </li><a name="line.1628"></a>
-<span class="sourceLineNo">1629</span> * <li>...if there are no subprocedure<a name="line.1629"></a>
-<span class="sourceLineNo">1630</span> * <ul><li>the procedure completed successfully<a name="line.1630"></a>
-<span class="sourceLineNo">1631</span> * <li>if there is a parent (WAITING)<a name="line.1631"></a>
-<span class="sourceLineNo">1632</span> * <li>the parent state will be set to RUNNABLE<a name="line.1632"></a>
-<span class="sourceLineNo">1633</span> * </ul><a name="line.1633"></a>
-<span class="sourceLineNo">1634</span> * </li><a name="line.1634"></a>
-<span class="sourceLineNo">1635</span> * </ul><a name="line.1635"></a>
-<span class="sourceLineNo">1636</span> * </li><a name="line.1636"></a>
-<span class="sourceLineNo">1637</span> * <li>In case of failure<a name="line.1637"></a>
-<span class="sourceLineNo">1638</span> * <ul><a name="line.1638"></a>
-<span class="sourceLineNo">1639</span> * <li>The store is updated with the new state</li><a name="line.1639"></a>
-<span class="sourceLineNo">1640</span> * <li>The executor (caller of this method) will start the rollback of the procedure</li><a name="line.1640"></a>
-<span class="sourceLineNo">1641</span> * </ul><a name="line.1641"></a>
-<span class="sourceLineNo">1642</span> * </li><a name="line.1642"></a>
-<span class="sourceLineNo">1643</span> * </ul><a name="line.1643"></a>
-<span class="sourceLineNo">1644</span> */<a name="line.1644"></a>
-<span class="sourceLineNo">1645</span> private void execProcedure(RootProcedureState<TEnvironment> procStack,<a name="line.1645"></a>
-<span class="sourceLineNo">1646</span> Procedure<TEnvironment> procedure) {<a name="line.1646"></a>
-<span class="sourceLineNo">1647</span> Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.1647"></a>
-<span class="sourceLineNo">1648</span> "NOT RUNNABLE! " + procedure.toString());<a name="line.1648"></a>
-<span class="sourceLineNo">1649</span><a name="line.1649"></a>
-<span class="sourceLineNo">1650</span> // Procedures can suspend themselves. They skip out by throwing a ProcedureSuspendedException.<a name="line.1650"></a>
-<span class="sourceLineNo">1651</span> // The exception is caught below and then we hurry to the exit without disturbing state. The<a name="line.1651"></a>
-<span class="sourceLineNo">1652</span> // idea is that the processing of this procedure will be unsuspended later by an external event<a name="line.1652"></a>
-<span class="sourceLineNo">1653</span> // such the report of a region open.<a name="line.1653"></a>
-<span class="sourceLineNo">1654</span> boolean suspended = false;<a name="line.1654"></a>
-<span class="sourceLineNo">1655</span><a name="line.1655"></a>
-<span class="sourceLineNo">1656</span> // Whether to 're-' -execute; run through the loop again.<a name="line.1656"></a>
-<span class="sourceLineNo">1657</span> boolean reExecute = false;<a name="line.1657"></a>
-<span class="sourceLineNo">1658</span><a name="line.1658"></a>
-<span class="sourceLineNo">1659</span> Procedure<TEnvironment>[] subprocs = null;<a name="line.1659"></a>
-<span class="sourceLineNo">1660</span> do {<a name="line.1660"></a>
-<span class="sourceLineNo">1661</span> reExecute = false;<a name="line.1661"></a>
-<span class="sourceLineNo">1662</span> procedure.resetPersistence();<a name="line.1662"></a>
-<span class="sourceLineNo">1663</span> try {<a name="line.1663"></a>
-<span class="sourceLineNo">1664</span> subprocs = procedure.doExecute(getEnvironment());<a name="line.1664"></a>
-<span class="sourceLineNo">1665</span> if (subprocs != null && subprocs.length == 0) {<a name="line.1665"></a>
-<span class="sourceLineNo">1666</span> subprocs = null;<a name="line.1666"></a>
-<span class="sourceLineNo">1667</span> }<a name="line.1667"></a>
-<span class="sourceLineNo">1668</span> } catch (ProcedureSuspendedException e) {<a name="line.1668"></a>
-<span class="sourceLineNo">1669</span> LOG.trace("Suspend {}", procedure);<a name="line.1669"></a>
-<span class="sourceLineNo">1670</span> suspended = true;<a name="line.1670"></a>
-<span class="sourceLineNo">1671</span> } catch (ProcedureYieldException e) {<a name="line.1671"></a>
-<span class="sourceLineNo">1672</span> LOG.trace("Yield {}", procedure, e);<a name="line.1672"></a>
-<span class="sourceLineNo">1673</span> yieldProcedure(procedure);<a name="line.1673"></a>
-<span class="sourceLineNo">1674</span> return;<a name="line.1674"></a>
-<span class="sourceLineNo">1675</span> } catch (InterruptedException e) {<a name="line.1675"></a>
-<span class="sourceLineNo">1676</span> LOG.trace("Yield interrupt {}", procedure, e);<a name="line.1676"></a>
-<span class="sourceLineNo">1677</span> handleInterruptedException(procedure, e);<a name="line.1677"></a>
-<span class="sourceLineNo">1678</span> yieldProcedure(procedure);<a name="line.1678"></a>
-<span class="sourceLineNo">1679</span> return;<a name="line.1679"></a>
-<span class="sourceLineNo">1680</span> } catch (Throwable e) {<a name="line.1680"></a>
-<span class="sourceLineNo">1681</span> // Catch NullPointerExceptions or similar errors...<a name="line.1681"></a>
-<span class="sourceLineNo">1682</span> String msg = "CODE-BUG: Uncaught runtime exception: " + procedure;<a name="line.1682"></a>
-<span class="sourceLineNo">1683</span> LOG.error(msg, e);<a name="line.1683"></a>
-<span class="sourceLineNo">1684</span> procedure.setFailure(new RemoteProcedureException(msg, e));<a name="line.1684"></a>
-<span class="sourceLineNo">1685</span> }<a name="line.1685"></a>
-<span class="sourceLineNo">1686</span><a name="line.1686"></a>
-<span class="sourceLineNo">1687</span> if (!procedure.isFailed()) {<a name="line.1687"></a>
-<span class="sourceLineNo">1688</span> if (subprocs != null) {<a name="line.1688"></a>
-<span class="sourceLineNo">1689</span> if (subprocs.length == 1 && subprocs[0] == procedure) {<a name="line.1689"></a>
-<span class="sourceLineNo">1690</span> // Procedure returned itself. Quick-shortcut for a state machine-like procedure;<a name="line.1690"></a>
-<span class="sourceLineNo">1691</span> // i.e. we go around this loop again rather than go back out on the scheduler queue.<a name="line.1691"></a>
-<span class="sourceLineNo">1692</span> subprocs = null;<a name="line.1692"></a>
-<span class="sourceLineNo">1693</span> reExecute = true;<a name="line.1693"></a>
-<span class="sourceLineNo">1694</span> LOG.trace("Short-circuit to next step on pid={}", procedure.getProcId());<a name="line.1694"></a>
-<span class="sourceLineNo">1695</span> } else {<a name="line.1695"></a>
-<span class="sourceLineNo">1696</span> // Yield the current procedure, and make the subprocedure runnable<a name="line.1696"></a>
-<span class="sourceLineNo">1697</span> // subprocs may come back 'null'.<a name="line.1697"></a>
-<span class="sourceLineNo">1698</span> subprocs = initializeChildren(procStack, procedure, subprocs);<a name="line.1698"></a>
-<span class="sourceLineNo">1699</span> LOG.info("Initialized subprocedures=" +<a name="line.1699"></a>
-<span class="sourceLineNo">1700</span> (subprocs == null? null:<a name="line.1700"></a>
-<span class="sourceLineNo">1701</span> Stream.of(subprocs).map(e -> "{" + e.toString() + "}").<a name="line.1701"></a>
-<span class="sourceLineNo">1702</span> collect(Collectors.toList()).toString()));<a name="line.1702"></a>
-<span class="sourceLineNo">1703</span> }<a name="line.1703"></a>
-<span class="sourceLineNo">1704</span> } else if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.1704"></a>
-<span class="sourceLineNo">1705</span> LOG.trace("Added to timeoutExecutor {}", procedure);<a name="line.1705"></a>
-<span class="sourceLineNo">1706</span> timeoutExecutor.add(procedure);<a name="line.1706"></a>
-<span class="sourceLineNo">1707</span> } else if (!suspended) {<a name="line.1707"></a>
-<span class="sourceLineNo">1708</span> // No subtask, so we are done<a name="line.1708"></a>
-<span class="sourceLineNo">1709</span> procedure.setState(ProcedureState.SUCCESS);<a name="line.1709"></a>
-<span class="sourceLineNo">1710</span> }<a name="line.1710"></a>
-<span class="sourceLineNo">1711</span> }<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span><a name="line.1712"></a>
-<span class="sourceLineNo">1713</span> // Add the procedure to the stack<a name="line.1713"></a>
-<span class="sourceLineNo">1714</span> procStack.addRollbackStep(procedure);<a name="line.1714"></a>
-<span class="sourceLineNo">1715</span><a name="line.1715"></a>
-<span class="sourceLineNo">1716</span> // allows to kill the executor before something is stored to the wal.<a name="line.1716"></a>
-<span class="sourceLineNo">1717</span> // useful to test the procedure recovery.<a name="line.1717"></a>
-<span class="sourceLineNo">1718</span> if (testing != null &&<a name="line.1718"></a>
-<span class="sourceLineNo">1719</span> testing.shouldKillBeforeStoreUpdate(suspended, procedure.hasParent())) {<a name="line.1719"></a>
-<span class="sourceLineNo">1720</span> kill("TESTING: Kill BEFORE store update: " + procedure);<a name="line.1720"></a>
-<span class="sourceLineNo">1721</span> }<a name="line.1721"></a>
+<span class="sourceLineNo">1553</span> // Finalize the procedure state<a name="line.1553"></a>
+<span class="sourceLineNo">1554</span> LOG.info("Rolled back {} exec-time={}", rootProc,<a name="line.1554"></a>
+<span class="sourceLineNo">1555</span> StringUtils.humanTimeDiff(rootProc.elapsedTime()));<a name="line.1555"></a>
+<span class="sourceLineNo">1556</span> procedureFinished(rootProc);<a name="line.1556"></a>
+<span class="sourceLineNo">1557</span> return LockState.LOCK_ACQUIRED;<a name="line.1557"></a>
+<span class="sourceLineNo">1558</span> }<a name="line.1558"></a>
+<span class="sourceLineNo">1559</span><a name="line.1559"></a>
+<span class="sourceLineNo">1560</span> private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) {<a name="line.1560"></a>
+<span class="sourceLineNo">1561</span> if (proc.removeStackIndex()) {<a name="line.1561"></a>
+<span class="sourceLineNo">1562</span> if (!proc.isSuccess()) {<a name="line.1562"></a>
+<span class="sourceLineNo">1563</span> proc.setState(ProcedureState.ROLLEDBACK);<a name="line.1563"></a>
+<span class="sourceLineNo">1564</span> }<a name="line.1564"></a>
+<span class="sourceLineNo">1565</span><a name="line.1565"></a>
+<span class="sourceLineNo">1566</span> // update metrics on finishing the procedure (fail)<a name="line.1566"></a>
+<span class="sourceLineNo">1567</span> proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false);<a name="line.1567"></a>
+<span class="sourceLineNo">1568</span><a name="line.1568"></a>
+<span class="sourceLineNo">1569</span> if (proc.hasParent()) {<a name="line.1569"></a>
+<span class="sourceLineNo">1570</span> store.delete(proc.getProcId());<a name="line.1570"></a>
+<span class="sourceLineNo">1571</span> procedures.remove(proc.getProcId());<a name="line.1571"></a>
+<span class="sourceLineNo">1572</span> } else {<a name="line.1572"></a>
+<span class="sourceLineNo">1573</span> final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds();<a name="line.1573"></a>
+<span class="sourceLineNo">1574</span> if (childProcIds != null) {<a name="line.1574"></a>
+<span class="sourceLineNo">1575</span> store.delete(proc, childProcIds);<a name="line.1575"></a>
+<span class="sourceLineNo">1576</span> } else {<a name="line.1576"></a>
+<span class="sourceLineNo">1577</span> store.update(proc);<a name="line.1577"></a>
+<span class="sourceLineNo">1578</span> }<a name="line.1578"></a>
+<span class="sourceLineNo">1579</span> }<a name="line.1579"></a>
+<span class="sourceLineNo">1580</span> } else {<a name="line.1580"></a>
+<span class="sourceLineNo">1581</span> store.update(proc);<a name="line.1581"></a>
+<span class="sourceLineNo">1582</span> }<a name="line.1582"></a>
+<span class="sourceLineNo">1583</span> }<a name="line.1583"></a>
+<span class="sourceLineNo">1584</span><a name="line.1584"></a>
+<span class="sourceLineNo">1585</span> /**<a name="line.1585"></a>
+<span class="sourceLineNo">1586</span> * Execute the rollback of the procedure step.<a name="line.1586"></a>
+<span class="sourceLineNo">1587</span> * It updates the store with the new state (stack index)<a name="line.1587"></a>
+<span class="sourceLineNo">1588</span> * or will remove completly the procedure in case it is a child.<a name="line.1588"></a>
+<span class="sourceLineNo">1589</span> */<a name="line.1589"></a>
+<span class="sourceLineNo">1590</span> private LockState executeRollback(Procedure<TEnvironment> proc) {<a name="line.1590"></a>
+<span class="sourceLineNo">1591</span> try {<a name="line.1591"></a>
+<span class="sourceLineNo">1592</span> proc.doRollback(getEnvironment());<a name="line.1592"></a>
+<span class="sourceLineNo">1593</span> } catch (IOException e) {<a name="line.1593"></a>
+<span class="sourceLineNo">1594</span> LOG.debug("Roll back attempt failed for {}", proc, e);<a name="line.1594"></a>
+<span class="sourceLineNo">1595</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1595"></a>
+<span class="sourceLineNo">1596</span> } catch (InterruptedException e) {<a name="line.1596"></a>
+<span class="sourceLineNo">1597</span> handleInterruptedException(proc, e);<a name="line.1597"></a>
+<span class="sourceLineNo">1598</span> return LockState.LOCK_YIELD_WAIT;<a name="line.1598"></a>
+<span class="sourceLineNo">1599</span> } catch (Throwable e) {<a name="line.1599"></a>
+<span class="sourceLineNo">1600</span> // Catch NullPointerExceptions or similar errors...<a name="line.1600"></a>
+<span class="sourceLineNo">1601</span> LOG.error(HBaseMarkers.FATAL, "CODE-BUG: Uncaught runtime exception for " + proc, e);<a name="line.1601"></a>
+<span class="sourceLineNo">1602</span> }<a name="line.1602"></a>
+<span class="sourceLineNo">1603</span><a name="line.1603"></a>
+<span class="sourceLineNo">1604</span> // allows to kill the executor before something is stored to the wal.<a name="line.1604"></a>
+<span class="sourceLineNo">1605</span> // useful to test the procedure recovery.<a name="line.1605"></a>
+<span class="sourceLineNo">1606</span> if (testing != null && testing.shouldKillBeforeStoreUpdate()) {<a name="line.1606"></a>
+<span class="sourceLineNo">1607</span> String msg = "TESTING: Kill before store update";<a name="line.1607"></a>
+<span class="sourceLineNo">1608</span> LOG.debug(msg);<a name="line.1608"></a>
+<span class="sourceLineNo">1609</span> stop();<a name="line.1609"></a>
+<span class="sourceLineNo">1610</span> throw new RuntimeException(msg);<a name="line.1610"></a>
+<span class="sourceLineNo">1611</span> }<a name="line.1611"></a>
+<span class="sourceLineNo">1612</span><a name="line.1612"></a>
+<span class="sourceLineNo">1613</span> cleanupAfterRollbackOneStep(proc);<a name="line.1613"></a>
+<span class="sourceLineNo">1614</span><a name="line.1614"></a>
+<span class="sourceLineNo">1615</span> return LockState.LOCK_ACQUIRED;<a name="line.1615"></a>
+<span class="sourceLineNo">1616</span> }<a name="line.1616"></a>
+<span class="sourceLineNo">1617</span><a name="line.1617"></a>
+<span class="sourceLineNo">1618</span> private void yieldProcedure(Procedure<TEnvironment> proc) {<a name="line.1618"></a>
+<span class="sourceLineNo">1619</span> releaseLock(proc, false);<a name="line.1619"></a>
+<span class="sourceLineNo">1620</span> scheduler.yield(proc);<a name="line.1620"></a>
+<span class="sourceLineNo">1621</span> }<a name="line.1621"></a>
+<span class="sourceLineNo">1622</span><a name="line.1622"></a>
+<span class="sourceLineNo">1623</span> /**<a name="line.1623"></a>
+<span class="sourceLineNo">1624</span> * Executes <code>procedure</code><a name="line.1624"></a>
+<span class="sourceLineNo">1625</span> * <ul><a name="line.1625"></a>
+<span class="sourceLineNo">1626</span> * <li>Calls the doExecute() of the procedure<a name="line.1626"></a>
+<span class="sourceLineNo">1627</span> * <li>If the procedure execution didn't fail (i.e. valid user input)<a name="line.1627"></a>
+<span class="sourceLineNo">1628</span> * <ul><a name="line.1628"></a>
+<span class="sourceLineNo">1629</span> * <li>...and returned subprocedures<a name="line.1629"></a>
+<span class="sourceLineNo">1630</span> * <ul><li>The subprocedures are initialized.<a name="line.1630"></a>
+<span class="sourceLineNo">1631</span> * <li>The subprocedures are added to the store<a name="line.1631"></a>
+<span class="sourceLineNo">1632</span> * <li>The subprocedures are added to the runnable queue<a name="line.1632"></a>
+<span class="sourceLineNo">1633</span> * <li>The procedure is now in a WAITING state, waiting for the subprocedures to complete<a name="line.1633"></a>
+<span class="sourceLineNo">1634</span> * </ul><a name="line.1634"></a>
+<span class="sourceLineNo">1635</span> * </li><a name="line.1635"></a>
+<span class="sourceLineNo">1636</span> * <li>...if there are no subprocedure<a name="line.1636"></a>
+<span class="sourceLineNo">1637</span> * <ul><li>the procedure completed successfully<a name="line.1637"></a>
+<span class="sourceLineNo">1638</span> * <li>if there is a parent (WAITING)<a name="line.1638"></a>
+<span class="sourceLineNo">1639</span> * <li>the parent state will be set to RUNNABLE<a name="line.1639"></a>
+<span class="sourceLineNo">1640</span> * </ul><a name="line.1640"></a>
+<span class="sourceLineNo">1641</span> * </li><a name="line.1641"></a>
+<span class="sourceLineNo">1642</span> * </ul><a name="line.1642"></a>
+<span class="sourceLineNo">1643</span> * </li><a name="line.1643"></a>
+<span class="sourceLineNo">1644</span> * <li>In case of failure<a name="line.1644"></a>
+<span class="sourceLineNo">1645</span> * <ul><a name="line.1645"></a>
+<span class="sourceLineNo">1646</span> * <li>The store is updated with the new state</li><a name="line.1646"></a>
+<span class="sourceLineNo">1647</span> * <li>The executor (caller of this method) will start the rollback of the procedure</li><a name="line.1647"></a>
+<span class="sourceLineNo">1648</span> * </ul><a name="line.1648"></a>
+<span class="sourceLineNo">1649</span> * </li><a name="line.1649"></a>
+<span class="sourceLineNo">1650</span> * </ul><a name="line.1650"></a>
+<span class="sourceLineNo">1651</span> */<a name="line.1651"></a>
+<span class="sourceLineNo">1652</span> private void execProcedure(RootProcedureState<TEnvironment> procStack,<a name="line.1652"></a>
+<span class="sourceLineNo">1653</span> Procedure<TEnvironment> procedure) {<a name="line.1653"></a>
+<span class="sourceLineNo">1654</span> Preconditions.checkArgument(procedure.getState() == ProcedureState.RUNNABLE,<a name="line.1654"></a>
+<span class="sourceLineNo">1655</span> "NOT RUNNABLE! " + procedure.toString());<a name="line.1655"></a>
+<span class="sourceLineNo">1656</span><a name="line.1656"></a>
+<span class="sourceLineNo">1657</span> // Procedures can suspend themselves. They skip out by throwing a ProcedureSuspendedException.<a name="line.1657"></a>
+<span class="sourceLineNo">1658</span> // The exception is caught below and then we hurry to the exit without disturbing state. The<a name="line.1658"></a>
+<span class="sourceLineNo">1659</span> // idea is that the processing of this procedure will be unsuspended later by an external event<a name="line.1659"></a>
+<span class="sourceLineNo">1660</span> // such the report of a region open.<a name="line.1660"></a>
+<span class="sourceLineNo">1661</span> boolean suspended = false;<a name="line.1661"></a>
+<span class="sourceLineNo">1662</span><a name="line.1662"></a>
+<span class="sourceLineNo">1663</span> // Whether to 're-' -execute; run through the loop again.<a name="line.1663"></a>
+<span class="sourceLineNo">1664</span> boolean reExecute = false;<a name="line.1664"></a>
+<span class="sourceLineNo">1665</span><a name="line.1665"></a>
+<span class="sourceLineNo">1666</span> Procedure<TEnvironment>[] subprocs = null;<a name="line.1666"></a>
+<span class="sourceLineNo">1667</span> do {<a name="line.1667"></a>
+<span class="sourceLineNo">1668</span> reExecute = false;<a name="line.1668"></a>
+<span class="sourceLineNo">1669</span> procedure.resetPersistence();<a name="line.1669"></a>
+<span class="sourceLineNo">1670</span> try {<a name="line.1670"></a>
+<span class="sourceLineNo">1671</span> subprocs = procedure.doExecute(getEnvironment());<a name="line.1671"></a>
+<span class="sourceLineNo">1672</span> if (subprocs != null && subprocs.length == 0) {<a name="line.1672"></a>
+<span class="sourceLineNo">1673</span> subprocs = null;<a name="line.1673"></a>
+<span class="sourceLineNo">1674</span> }<a name="line.1674"></a>
+<span class="sourceLineNo">1675</span> } catch (ProcedureSuspendedException e) {<a name="line.1675"></a>
+<span class="sourceLineNo">1676</span> LOG.trace("Suspend {}", procedure);<a name="line.1676"></a>
+<span class="sourceLineNo">1677</span> suspended = true;<a name="line.1677"></a>
+<span class="sourceLineNo">1678</span> } catch (ProcedureYieldException e) {<a name="line.1678"></a>
+<span class="sourceLineNo">1679</span> LOG.trace("Yield {}", procedure, e);<a name="line.1679"></a>
+<span class="sourceLineNo">1680</span> yieldProcedure(procedure);<a name="line.1680"></a>
+<span class="sourceLineNo">1681</span> return;<a name="line.1681"></a>
+<span class="sourceLineNo">1682</span> } catch (InterruptedException e) {<a name="line.1682"></a>
+<span class="sourceLineNo">1683</span> LOG.trace("Yield interrupt {}", procedure, e);<a name="line.1683"></a>
+<span class="sourceLineNo">1684</span> handleInterruptedException(procedure, e);<a name="line.1684"></a>
+<span class="sourceLineNo">1685</span> yieldProcedure(procedure);<a name="line.1685"></a>
+<span class="sourceLineNo">1686</span> return;<a name="line.1686"></a>
+<span class="sourceLineNo">1687</span> } catch (Throwable e) {<a name="line.1687"></a>
+<span class="sourceLineNo">1688</span> // Catch NullPointerExceptions or similar errors...<a name="line.1688"></a>
+<span class="sourceLineNo">1689</span> String msg = "CODE-BUG: Uncaught runtime exception: " + procedure;<a name="line.1689"></a>
+<span class="sourceLineNo">1690</span> LOG.error(msg, e);<a name="line.1690"></a>
+<span class="sourceLineNo">1691</span> procedure.setFailure(new RemoteProcedureException(msg, e));<a name="line.1691"></a>
+<span class="sourceLineNo">1692</span> }<a name="line.1692"></a>
+<span class="sourceLineNo">1693</span><a name="line.1693"></a>
+<span class="sourceLineNo">1694</span> if (!procedure.isFailed()) {<a name="line.1694"></a>
+<span class="sourceLineNo">1695</span> if (subprocs != null) {<a name="line.1695"></a>
+<span class="sourceLineNo">1696</span> if (subprocs.length == 1 && subprocs[0] == procedure) {<a name="line.1696"></a>
+<span class="sourceLineNo">1697</span> // Procedure returned itself. Quick-shortcut for a state machine-like procedure;<a name="line.1697"></a>
+<span class="sourceLineNo">1698</span> // i.e. we go around this loop again rather than go back out on the scheduler queue.<a name="line.1698"></a>
+<span class="sourceLineNo">1699</span> subprocs = null;<a name="line.1699"></a>
+<span class="sourceLineNo">1700</span> reExecute = true;<a name="line.1700"></a>
+<span class="sourceLineNo">1701</span> LOG.trace("Short-circuit to next step on pid={}", procedure.getProcId());<a name="line.1701"></a>
+<span class="sourceLineNo">1702</span> } else {<a name="line.1702"></a>
+<span class="sourceLineNo">1703</span> // Yield the current procedure, and make the subprocedure runnable<a name="line.1703"></a>
+<span class="sourceLineNo">1704</span> // subprocs may come back 'null'.<a name="line.1704"></a>
+<span class="sourceLineNo">1705</span> subprocs = initializeChildren(procStack, procedure, subprocs);<a name="line.1705"></a>
+<span class="sourceLineNo">1706</span> LOG.info("Initialized subprocedures=" +<a name="line.1706"></a>
+<span class="sourceLineNo">1707</span> (subprocs == null? null:<a name="line.1707"></a>
+<span class="sourceLineNo">1708</span> Stream.of(subprocs).map(e -> "{" + e.toString() + "}").<a name="line.1708"></a>
+<span class="sourceLineNo">1709</span> collect(Collectors.toList()).toString()));<a name="line.1709"></a>
+<span class="sourceLineNo">1710</span> }<a name="line.1710"></a>
+<span class="sourceLineNo">1711</span> } else if (procedure.getState() == ProcedureState.WAITING_TIMEOUT) {<a name="line.1711"></a>
+<span class="sourceLineNo">1712</span> LOG.trace("Added to timeoutExecutor {}", procedure);<a name="line.1712"></a>
+<span class="sourceLineNo">1713</span> timeoutExecutor.add(procedure);<a name="line.1713"></a>
+<span class="sourceLineNo">1714</span> } else if (!suspended) {<a name="line.1714"></a>
+<span class="sourceLineNo">1715</span> // No subtask, so we are done<a name="line.1715"></a>
+<span class="sourceLineNo">1716</span> procedure.setState(ProcedureState.SUCCESS);<a name="line.1716"></a>
+<span class="sourceLineNo">1717</span> }<a name="line.1717"></a>
+<span class="sourceLineNo">1718</span> }<a name="line.1718"></a>
+<span class="sourceLineNo">1719</span><a name="line.1719"></a>
+<span class="sourceLineNo">1720</span> // Add the procedure to the stack<a name="line.1720"></a>
+<span class="sourceLineNo">1721</span> procStack.addRollbackStep(procedure);<a name="line.1721"></a>
<span class="sourceLineNo">1722</span><a name="line.1722"></a>
-<span class="sourceLineNo">1723</span> // TODO: The code here doesn't check if store is running before persisting to the store as<a name="line.1723"></a>
-<span class="sourceLineNo">1724</span> // it relies on the method call below to throw RuntimeException to wind up the stack and<a name="line.1724"></a>
-<span class="sourceLineNo">1725</span> // executor thread to stop. The statement following the method call below seems to check if<a name="line.1725"></a>
-<span class="sourceLineNo">1726</span> // store is not running, to prevent scheduling children procedures, re-execution or yield<a name="line.1726"></a>
-<span class="sourceLineNo">1727</span> // of this procedure. This may need more scrutiny and subsequent cleanup in future<a name="line.1727"></a>
-<span class="sourceLineNo">1728</span> //<a name="line.1728"></a>
-<span class="sourceLineNo">1729</span> // Commit the transaction even if a suspend (state may have changed). Note this append<a name="line.1729"></a>
-<span class="sourceLineNo">1730</span> // can take a bunch of time to complete.<a name="line.1730"></a>
-<span class="sourceLineNo">1731</span> if (procedure.needPersistence()) {<a name="line.1731"></a>
-<span class="sourceLineNo">1732</span> updateStoreOnExec(procStack, procedure, subprocs);<a name="line.1732"></a>
-<span class="sourceLineNo">1733</span> }<a name="line.1733"></a>
-<span class="sourceLineNo">1734</span><a name="line.1734"></a>
-<span class="sourceLineNo">1735</span> // if the store is not running we are aborting<a name="line.1735"></a>
-<span class="sourceLineNo">1736</span> if (!store.isRunning()) {<a name="line.1736"></a>
-<span class="sourceLineNo">1737</span> return;<a name="line.1737"></a>
-<span class="sourceLineNo">1738</span> }<a name="line.1738"></a>
-<span class="sourceLineNo">1739</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1739"></a>
-<span class="sourceLineNo">1740</span> if (procedure.isRunnable() && !suspended &&<a name="line.1740"></a>
-<span class="sourceLineNo">1741</span> procedure.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1741"></a>
-<span class="sourceLineNo">1742</span> yieldProcedure(procedure);<a name="line.1742"></a>
-<span class="sourceLineNo">1743</span> return;<a name="line.1743"></a>
-<span class="sourceLineNo">1744</span> }<a name="line.1744"></a>
-<span class="sourceLineNo">1745</span><a name="line.1745"></a>
-<span class="sourceLineNo">1746</span> assert (reExecute && subprocs == null) || !reExecute;<a name="line.1746"></a>
-<span class="sourceLineNo">1747</span> } while (reExecute);<a name="line.1747"></a>
-<span class="sourceLineNo">1748</span><a name="line.1748"></a>
-<span class="sourceLineNo">1749</span> // Allows to kill the executor after something is stored to the WAL but before the below<a name="line.1749"></a>
-<span class="sourceLineNo">1750</span> // state settings are done -- in particular the one on the end where we make parent<a name="line.1750"></a>
-<span class="sourceLineNo">1751</span> // RUNNABLE again when its children are done; see countDownChildren.<a name="line.1751"></a>
-<span class="sourceLineNo">1752</span> if (testing != null && testing.shouldKillAfterStoreUpdate(suspended)) {<a name="line.1752"></a>
-<span class="sourceLineNo">1753</span> kill("TESTING: Kill AFTER store update: " + procedure);<a name="line.1753"></a>
-<span class="sourceLineNo">1754</span> }<a name="line.1754"></a>
+<span class="sourceLineNo">1723</span> // allows to kill the executor before something is stored to the wal.<a name="line.1723"></a>
+<span class="sourceLineNo">1724</span> // useful to test the procedure recovery.<a name="line.1724"></a>
+<span class="sourceLineNo">1725</span> if (testing != null &&<a name="line.1725"></a>
+<span class="sourceLineNo">1726</span> testing.shouldKillBeforeStoreUpdate(suspended, procedure.hasParent())) {<a name="line.1726"></a>
+<span class="sourceLineNo">1727</span> kill("TESTING: Kill BEFORE store update: " + procedure);<a name="line.1727"></a>
+<span class="sourceLineNo">1728</span> }<a name="line.1728"></a>
+<span class="sourceLineNo">1729</span><a name="line.1729"></a>
+<span class="sourceLineNo">1730</span> // TODO: The code here doesn't check if store is running before persisting to the store as<a name="line.1730"></a>
+<span class="sourceLineNo">1731</span> // it relies on the method call below to throw RuntimeException to wind up the stack and<a name="line.1731"></a>
+<span class="sourceLineNo">1732</span> // executor thread to stop. The statement following the method call below seems to check if<a name="line.1732"></a>
+<span class="sourceLineNo">1733</span> // store is not running, to prevent scheduling children procedures, re-execution or yield<a name="line.1733"></a>
+<span class="sourceLineNo">1734</span> // of this procedure. This may need more scrutiny and subsequent cleanup in future<a name="line.1734"></a>
+<span class="sourceLineNo">1735</span> //<a name="line.1735"></a>
+<span class="sourceLineNo">1736</span> // Commit the transaction even if a suspend (state may have changed). Note this append<a name="line.1736"></a>
+<span class="sourceLineNo">1737</span> // can take a bunch of time to complete.<a name="line.1737"></a>
+<span class="sourceLineNo">1738</span> if (procedure.needPersistence()) {<a name="line.1738"></a>
+<span class="sourceLineNo">1739</span> updateStoreOnExec(procStack, procedure, subprocs);<a name="line.1739"></a>
+<span class="sourceLineNo">1740</span> }<a name="line.1740"></a>
+<span class="sourceLineNo">1741</span><a name="line.1741"></a>
+<span class="sourceLineNo">1742</span> // if the store is not running we are aborting<a name="line.1742"></a>
+<span class="sourceLineNo">1743</span> if (!store.isRunning()) {<a name="line.1743"></a>
+<span class="sourceLineNo">1744</span> return;<a name="line.1744"></a>
+<span class="sourceLineNo">1745</span> }<a name="line.1745"></a>
+<span class="sourceLineNo">1746</span> // if the procedure is kind enough to pass the slot to someone else, yield<a name="line.1746"></a>
+<span class="sourceLineNo">1747</span> if (procedure.isRunnable() && !suspended &&<a name="line.1747"></a>
+<span class="sourceLineNo">1748</span> procedure.isYieldAfterExecutionStep(getEnvironment())) {<a name="line.1748"></a>
+<span class="sourceLineNo">1749</span> yieldProcedure(procedure);<a name="line.1749"></a>
+<span class="sourceLineNo">1750</span> return;<a name="line.1750"></a>
+<span class="sourceLineNo">1751</span> }<a name="line.1751"></a>
+<span class="sourceLineNo">1752</span><a name="line.1752"></a>
+<span class="sourceLineNo">1753</span> assert (reExecute && subprocs == null) || !reExecute;<a name="line.1753"></a>
+<span class="sourceLineNo">1754</span> } while (reExecute);<a name="line.1754"></a>
<span class="sourceLineNo">1755</span><a name="line.1755"></a>
-<span class="sourceLineNo">1756</span> // Submit the new subprocedures<a name="line.1756"></a>
-<span class="sourceLineNo">1757</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1757"></a>
-<span class="sourceLineNo">1758</span> submitChildrenProcedures(subprocs);<a name="line.1758"></a>
-<span class="sourceLineNo">1759</span> }<a name="line.1759"></a>
-<span class="sourceLineNo">1760</span><a name="line.1760"></a>
-<span class="sourceLineNo">1761</span> // we need to log the release lock operation before waking up the parent procedure, as there<a name="line.1761"></a>
-<span class="sourceLineNo">1762</span> // could be race that the parent procedure may call updateStoreOnExec ahead of us and remove all<a name="line.1762"></a>
-<span class="sourceLineNo">1763</span> // the sub procedures from store and cause problems...<a name="line.1763"></a>
-<span class="sourceLineNo">1764</span> releaseLock(procedure, false);<a name="line.1764"></a>
-<span class="sourceLineNo">1765</span><a name="line.1765"></a>
-<span class="sourceLineNo">1766</span> // if the procedure is complete and has a parent, count down the children latch.<a name="line.1766"></a>
-<span class="sourceLineNo">1767</span> // If 'suspended', do nothing to change state -- let other threads handle unsuspend event.<a name="line.1767"></a>
-<span class="sourceLineNo">1768</span> if (!suspended && procedure.isFinished() && procedure.hasParent()) {<a name="line.1768"></a>
-<span class="sourceLineNo">1769</span> countDownChildren(procStack, procedure);<a name="line.1769"></a>
-<span class="sourceLineNo">1770</span> }<a name="line.1770"></a>
-<span class="sourceLineNo">1771</span> }<a name="line.1771"></a>
+<span class="sourceLineNo">1756</span> // Allows to kill the executor after something is stored to the WAL but before the below<a name="line.1756"></a>
+<span class="sourceLineNo">1757</span> // state settings are done -- in particular the one on the end where we make parent<a name="line.1757"></a>
+<span class="sourceLineNo">1758</span> // RUNNABLE again when its children are done; see countDownChildren.<a name="line.1758"></a>
+<span class="sourceLineNo">1759</span> if (testing != null && testing.shouldKillAfterStoreUpdate(suspended)) {<a name="line.1759"></a>
+<span class="sourceLineNo">1760</span> kill("TESTING: Kill AFTER store update: " + procedure);<a name="line.1760"></a>
+<span class="sourceLineNo">1761</span> }<a name="line.1761"></a>
+<span class="sourceLineNo">1762</span><a name="line.1762"></a>
+<span class="sourceLineNo">1763</span> // Submit the new subprocedures<a name="line.1763"></a>
+<span class="sourceLineNo">1764</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1764"></a>
+<span class="sourceLineNo">1765</span> submitChildrenProcedures(subprocs);<a name="line.1765"></a>
+<span class="sourceLineNo">1766</span> }<a name="line.1766"></a>
+<span class="sourceLineNo">1767</span><a name="line.1767"></a>
+<span class="sourceLineNo">1768</span> // we need to log the release lock operation before waking up the parent procedure, as there<a name="line.1768"></a>
+<span class="sourceLineNo">1769</span> // could be race that the parent procedure may call updateStoreOnExec ahead of us and remove all<a name="line.1769"></a>
+<span class="sourceLineNo">1770</span> // the sub procedures from store and cause problems...<a name="line.1770"></a>
+<span class="sourceLineNo">1771</span> releaseLock(procedure, false);<a name="line.1771"></a>
<span class="sourceLineNo">1772</span><a name="line.1772"></a>
-<span class="sourceLineNo">1773</span> private void kill(String msg) {<a name="line.1773"></a>
-<span class="sourceLineNo">1774</span> LOG.debug(msg);<a name="line.1774"></a>
-<span class="sourceLineNo">1775</span> stop();<a name="line.1775"></a>
-<span class="sourceLineNo">1776</span> throw new RuntimeException(msg);<a name="line.1776"></a>
-<span class="sourceLineNo">1777</span> }<a name="line.1777"></a>
-<span class="sourceLineNo">1778</span><a name="line.1778"></a>
-<span class="sourceLineNo">1779</span> private Procedure<TEnvironment>[] initializeChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1779"></a>
-<span class="sourceLineNo">1780</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1780"></a>
-<span class="sourceLineNo">1781</span> assert subprocs != null : "expected subprocedures";<a name="line.1781"></a>
-<span class="sourceLineNo">1782</span> final long rootProcId = getRootProcedureId(procedure);<a name="line.1782"></a>
-<span class="sourceLineNo">1783</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1783"></a>
-<span class="sourceLineNo">1784</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1784"></a>
-<span class="sourceLineNo">1785</span> if (subproc == null) {<a name="line.1785"></a>
-<span class="sourceLineNo">1786</span> String msg = "subproc[" + i + "] is null, aborting the procedure";<a name="line.1786"></a>
-<span class="sourceLineNo">1787</span> procedure.setFailure(new RemoteProcedureException(msg,<a name="line.1787"></a>
-<span class="sourceLineNo">1788</span> new IllegalArgumentIOException(msg)));<a name="line.1788"></a>
-<span class="sourceLineNo">1789</span> return null;<a name="line.1789"></a>
-<span class="sourceLineNo">1790</span> }<a name="line.1790"></a>
-<span class="sourceLineNo">1791</span><a name="line.1791"></a>
-<span class="sourceLineNo">1792</span> assert subproc.getState() == ProcedureState.INITIALIZING : subproc;<a name="line.1792"></a>
-<span class="sourceLineNo">1793</span> subproc.setParentProcId(procedure.getProcId());<a name="line.1793"></a>
-<span class="sourceLineNo">1794</span> subproc.setRootProcId(rootProcId);<a name="line.1794"></a>
-<span class="sourceLineNo">1795</span> subproc.setProcId(nextProcId());<a name="line.1795"></a>
-<span class="sourceLineNo">1796</span> procStack.addSubProcedure(subproc);<a name="line.1796"></a>
-<span class="sourceLineNo">1797</span> }<a name="line.1797"></a>
+<span class="sourceLineNo">1773</span> // if the procedure is complete and has a parent, count down the children latch.<a name="line.1773"></a>
+<span class="sourceLineNo">1774</span> // If 'suspended', do nothing to change state -- let other threads handle unsuspend event.<a name="line.1774"></a>
+<span class="sourceLineNo">1775</span> if (!suspended && procedure.isFinished() && procedure.hasParent()) {<a name="line.1775"></a>
+<span class="sourceLineNo">1776</span> countDownChildren(procStack, procedure);<a name="line.1776"></a>
+<span class="sourceLineNo">1777</span> }<a name="line.1777"></a>
+<span class="sourceLineNo">1778</span> }<a name="line.1778"></a>
+<span class="sourceLineNo">1779</span><a name="line.1779"></a>
+<span class="sourceLineNo">1780</span> private void kill(String msg) {<a name="line.1780"></a>
+<span class="sourceLineNo">1781</span> LOG.debug(msg);<a name="line.1781"></a>
+<span class="sourceLineNo">1782</span> stop();<a name="line.1782"></a>
+<span class="sourceLineNo">1783</span> throw new RuntimeException(msg);<a name="line.1783"></a>
+<span class="sourceLineNo">1784</span> }<a name="line.1784"></a>
+<span class="sourceLineNo">1785</span><a name="line.1785"></a>
+<span class="sourceLineNo">1786</span> private Procedure<TEnvironment>[] initializeChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1786"></a>
+<span class="sourceLineNo">1787</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1787"></a>
+<span class="sourceLineNo">1788</span> assert subprocs != null : "expected subprocedures";<a name="line.1788"></a>
+<span class="sourceLineNo">1789</span> final long rootProcId = getRootProcedureId(procedure);<a name="line.1789"></a>
+<span class="sourceLineNo">1790</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1790"></a>
+<span class="sourceLineNo">1791</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1791"></a>
+<span class="sourceLineNo">1792</span> if (subproc == null) {<a name="line.1792"></a>
+<span class="sourceLineNo">1793</span> String msg = "subproc[" + i + "] is null, aborting the procedure";<a name="line.1793"></a>
+<span class="sourceLineNo">1794</span> procedure.setFailure(new RemoteProcedureException(msg,<a name="line.1794"></a>
+<span class="sourceLineNo">1795</span> new IllegalArgumentIOException(msg)));<a name="line.1795"></a>
+<span class="sourceLineNo">1796</span> return null;<a name="line.1796"></a>
+<span class="sourceLineNo">1797</span> }<a name="line.1797"></a>
<span class="sourceLineNo">1798</span><a name="line.1798"></a>
-<span class="sourceLineNo">1799</span> if (!procedure.isFailed()) {<a name="line.1799"></a>
-<span class="sourceLineNo">1800</span> procedure.setChildrenLatch(subprocs.length);<a name="line.1800"></a>
-<span class="sourceLineNo">1801</span> switch (procedure.getState()) {<a name="line.1801"></a>
-<span class="sourceLineNo">1802</span> case RUNNABLE:<a name="line.1802"></a>
-<span class="sourceLineNo">1803</span> procedure.setState(ProcedureState.WAITING);<a name="line.1803"></a>
-<span class="sourceLineNo">1804</span> break;<a name="line.1804"></a>
-<span class="sourceLineNo">1805</span> case WAITING_TIMEOUT:<a name="line.1805"></a>
-<span class="sourceLineNo">1806</span> timeoutExecutor.add(procedure);<a name="line.1806"></a>
-<span class="sourceLineNo">1807</span> break;<a name="line.1807"></a>
-<span class="sourceLineNo">1808</span> default:<a name="line.1808"></a>
-<span class="sourceLineNo">1809</span> break;<a name="line.1809"></a>
-<span class="sourceLineNo">1810</span> }<a name="line.1810"></a>
-<span class="sourceLineNo">1811</span> }<a name="line.1811"></a>
-<span class="sourceLineNo">1812</span> return subprocs;<a name="line.1812"></a>
-<span class="sourceLineNo">1813</span> }<a name="line.1813"></a>
-<span class="sourceLineNo">1814</span><a name="line.1814"></a>
-<span class="sourceLineNo">1815</span> private void submitChildrenProcedures(Procedure<TEnvironment>[] subprocs) {<a name="line.1815"></a>
-<span class="sourceLineNo">1816</span> for (int i = 0; i < subprocs.length; ++i) {<a name="line.1816"></a>
-<span class="sourceLineNo">1817</span> Procedure<TEnvironment> subproc = subprocs[i];<a name="line.1817"></a>
-<span class="sourceLineNo">1818</span> subproc.updateMetricsOnSubmit(getEnvironment());<a name="line.1818"></a>
-<span class="sourceLineNo">1819</span> assert !procedures.containsKey(subproc.getProcId());<a name="line.1819"></a>
-<span class="sourceLineNo">1820</span> procedures.put(subproc.getProcId(), subproc);<a name="line.1820"></a>
-<span class="sourceLineNo">1821</span> scheduler.addFront(subproc);<a name="line.1821"></a>
-<span class="sourceLineNo">1822</span> }<a name="line.1822"></a>
-<span class="sourceLineNo">1823</span> }<a name="line.1823"></a>
-<span class="sourceLineNo">1824</span><a name="line.1824"></a>
-<span class="sourceLineNo">1825</span> private void countDownChildren(RootProcedureState<TEnvironment> procStack,<a name="line.1825"></a>
-<span class="sourceLineNo">1826</span> Procedure<TEnvironment> procedure) {<a name="line.1826"></a>
-<span class="sourceLineNo">1827</span> Procedure<TEnvironment> parent = procedures.get(procedure.getParentProcId());<a name="line.1827"></a>
-<span class="sourceLineNo">1828</span> if (parent == null) {<a name="line.1828"></a>
-<span class="sourceLineNo">1829</span> assert procStack.isRollingback();<a name="line.1829"></a>
-<span class="sourceLineNo">1830</span> return;<a name="line.1830"></a>
-<span class="sourceLineNo">1831</span> }<a name="line.1831"></a>
-<span class="sourceLineNo">1832</span><a name="line.1832"></a>
-<span class="sourceLineNo">1833</span> // If this procedure is the last child awake the parent procedure<a name="line.1833"></a>
-<span class="sourceLineNo">1834</span> if (parent.tryRunnable()) {<a name="line.1834"></a>
-<span class="sourceLineNo">1835</span> // If we succeeded in making the parent runnable -- i.e. all of its<a name="line.1835"></a>
-<span class="sourceLineNo">1836</span> // children have completed, move parent to front of the queue.<a name="line.1836"></a>
-<span class="sourceLineNo">1837</span> store.update(parent);<a name="line.1837"></a>
-<span class="sourceLineNo">1838</span> scheduler.addFront(parent);<a name="line.1838"></a>
-<span class="sourceLineNo">1839</span> LOG.info("Finished subprocedure(s) of " + parent + "; resume parent processing.");<a name="line.1839"></a>
-<span class="sourceLineNo">1840</span> return;<a name="line.1840"></a>
-<span class="sourceLineNo">1841</span> }<a name="line.1841"></a>
-<span class="sourceLineNo">1842</span> }<a name="line.1842"></a>
-<span class="sourceLineNo">1843</span><a name="line.1843"></a>
-<span class="sourceLineNo">1844</span> private void updateStoreOnExec(RootProcedureState<TEnvironment> procStack,<a name="line.1844"></a>
-<span class="sourceLineNo">1845</span> Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) {<a name="line.1845"></a>
-<span class="sourceLineNo">1846</span> if (subprocs != null && !procedure.isFailed()) {<a name="line.1846"></a>
-<span class="sourceLineNo">1847</span> if (LOG.isTraceEnabled()) {<a name="line.1847"></a>
-<span class="sourceLineNo">1848</span> LOG.trace("Stored " + procedure + ", children " + Arrays.toString(subprocs));<a name="line.1848"></a>
-<span class="sourceLineNo">1849</span> }<a name="line.1849"></a>
-<span class="sourceLineNo">1850</span> store.insert(procedure, subprocs);<a name="line.1850"></a>
-<span class="sourceLineNo">1851</span> } else {<a name="line.1851"></a>
-<span class="sourceLineNo">1852</span> LOG.trace("Store update {}", procedure);<a name="line.1852"></a>
-<span class="sourceLineNo">1853</span> if (procedure.isFinished() && !procedure.hasParent()) {<a name="line.1853"></a>
-<span class="sourceLineNo">1854</span> // remove child procedures<a name="line.1854"></a>
-<span class="sourceLineNo">1855</span> final long[] childProcIds = procStack.getSubprocedureIds();<a name="line.1855"></a>
-<span class="sourceLineNo">1856</span> if (childProcIds != null) {<a name="line.1856"></a>
-<span class="sourceLineNo">1857</span> store.delete(procedure, childProcIds);<a name="line.1857"></a>
-<span class="sourceLineNo">1858</span> for (int i = 0; i < childProcIds.length; ++i) {<a name="line.1858"></a>
... 19179 lines suppressed ...