You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/01/20 15:31:36 UTC
[16/28] hbase-site git commit: Published site at .
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
index 7271567..a08bfa4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.ProcedureExecutorListener.html
@@ -518,10 +518,10 @@
<span class="sourceLineNo">510</span> // We have numThreads executor + one timer thread used for timing out<a name="line.510"></a>
<span class="sourceLineNo">511</span> // procedures and triggering periodic procedures.<a name="line.511"></a>
<span class="sourceLineNo">512</span> this.corePoolSize = numThreads;<a name="line.512"></a>
-<span class="sourceLineNo">513</span> LOG.info("Starting ProcedureExecutor Worker threads (ProcExecWrkr)=" + corePoolSize);<a name="line.513"></a>
+<span class="sourceLineNo">513</span> LOG.info("Starting ProcedureExecutor Worker threads (ProcedureExecutorWorker)=" + corePoolSize);<a name="line.513"></a>
<span class="sourceLineNo">514</span><a name="line.514"></a>
<span class="sourceLineNo">515</span> // Create the Thread Group for the executors<a name="line.515"></a>
-<span class="sourceLineNo">516</span> threadGroup = new ThreadGroup("ProcExecThrdGrp");<a name="line.516"></a>
+<span class="sourceLineNo">516</span> threadGroup = new ThreadGroup("ProcedureExecutorWorkerGroup");<a name="line.516"></a>
<span class="sourceLineNo">517</span><a name="line.517"></a>
<span class="sourceLineNo">518</span> // Create the timeout executor<a name="line.518"></a>
<span class="sourceLineNo">519</span> timeoutExecutor = new TimeoutExecutorThread(threadGroup);<a name="line.519"></a>
@@ -600,7 +600,7 @@
<span class="sourceLineNo">592</span> try {<a name="line.592"></a>
<span class="sourceLineNo">593</span> threadGroup.destroy();<a name="line.593"></a>
<span class="sourceLineNo">594</span> } catch (IllegalThreadStateException e) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span> LOG.error("Thread group " + threadGroup + " contains running threads");<a name="line.595"></a>
+<span class="sourceLineNo">595</span> LOG.error("ThreadGroup " + threadGroup + " contains running threads; " + e.getMessage());<a name="line.595"></a>
<span class="sourceLineNo">596</span> threadGroup.list();<a name="line.596"></a>
<span class="sourceLineNo">597</span> } finally {<a name="line.597"></a>
<span class="sourceLineNo">598</span> threadGroup = null;<a name="line.598"></a>
@@ -1717,7 +1717,7 @@
<span class="sourceLineNo">1709</span> private Procedure activeProcedure;<a name="line.1709"></a>
<span class="sourceLineNo">1710</span><a name="line.1710"></a>
<span class="sourceLineNo">1711</span> public WorkerThread(final ThreadGroup group) {<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span> super(group, "ProcExecWrkr-" + workerId.incrementAndGet());<a name="line.1712"></a>
+<span class="sourceLineNo">1712</span> super(group, "ProcedureExecutorWorker-" + workerId.incrementAndGet());<a name="line.1712"></a>
<span class="sourceLineNo">1713</span> setDaemon(true);<a name="line.1713"></a>
<span class="sourceLineNo">1714</span> }<a name="line.1714"></a>
<span class="sourceLineNo">1715</span><a name="line.1715"></a>
@@ -1760,7 +1760,7 @@
<span class="sourceLineNo">1752</span> } catch (Throwable t) {<a name="line.1752"></a>
<span class="sourceLineNo">1753</span> LOG.warn("Worker terminating UNNATURALLY " + this.activeProcedure, t);<a name="line.1753"></a>
<span class="sourceLineNo">1754</span> } finally {<a name="line.1754"></a>
-<span class="sourceLineNo">1755</span> LOG.debug("Worker terminated.");<a name="line.1755"></a>
+<span class="sourceLineNo">1755</span> LOG.trace("Worker terminated.");<a name="line.1755"></a>
<span class="sourceLineNo">1756</span> }<a name="line.1756"></a>
<span class="sourceLineNo">1757</span> workerThreads.remove(this);<a name="line.1757"></a>
<span class="sourceLineNo">1758</span> }<a name="line.1758"></a>
@@ -1912,120 +1912,123 @@
<span class="sourceLineNo">1904</span> for (int i = 0; isAlive(); ++i) {<a name="line.1904"></a>
<span class="sourceLineNo">1905</span> sendStopSignal();<a name="line.1905"></a>
<span class="sourceLineNo">1906</span> join(250);<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span> if (i > 0 && (i % 8) == 0) {<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span> LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span> StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime));<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span> }<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span> }<a name="line.1911"></a>
-<span class="sourceLineNo">1912</span> } catch (InterruptedException e) {<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span> LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span> }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span> }<a name="line.1915"></a>
-<span class="sourceLineNo">1916</span> }<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span><a name="line.1917"></a>
-<span class="sourceLineNo">1918</span> // ==========================================================================<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span> // Inline Chores (executors internal chores)<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span> // ==========================================================================<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span> private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span> private long timeout;<a name="line.1922"></a>
-<span class="sourceLineNo">1923</span><a name="line.1923"></a>
-<span class="sourceLineNo">1924</span> public abstract int getTimeoutInterval();<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span><a name="line.1925"></a>
-<span class="sourceLineNo">1926</span> protected void refreshTimeout() {<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span> this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span> }<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span><a name="line.1929"></a>
-<span class="sourceLineNo">1930</span> @Override<a name="line.1930"></a>
-<span class="sourceLineNo">1931</span> public long getTimeout() {<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span> return timeout;<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span> }<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span> }<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span><a name="line.1935"></a>
-<span class="sourceLineNo">1936</span> // ----------------------------------------------------------------------------<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1937"></a>
-<span class="sourceLineNo">1938</span> // full set of procedures pending and completed to write a compacted<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span> // version of the log (in case is a log)?<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span> // will have the tracker saying everything is in the last log.<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span> // ----------------------------------------------------------------------------<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span><a name="line.1943"></a>
-<span class="sourceLineNo">1944</span> private final class WorkerMonitor extends InlineChore {<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span><a name="line.1948"></a>
-<span class="sourceLineNo">1949</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span><a name="line.1952"></a>
-<span class="sourceLineNo">1953</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span><a name="line.1956"></a>
-<span class="sourceLineNo">1957</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span><a name="line.1960"></a>
-<span class="sourceLineNo">1961</span> public WorkerMonitor() {<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span> refreshConfig();<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span> }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span><a name="line.1964"></a>
-<span class="sourceLineNo">1965</span> @Override<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span> public void run() {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span> final int stuckCount = checkForStuckWorkers();<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span> checkThreadCount(stuckCount);<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span><a name="line.1969"></a>
-<span class="sourceLineNo">1970</span> // refresh interval (poor man dynamic conf update)<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span> refreshConfig();<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span> }<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span><a name="line.1973"></a>
-<span class="sourceLineNo">1974</span> private int checkForStuckWorkers() {<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span> // check if any of the worker is stuck<a name="line.1975"></a>
-<span class="sourceLineNo">1976</span> int stuckCount = 0;<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span> for (WorkerThread worker: workerThreads) {<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span> continue;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span> }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span> // WARN the worker is stuck<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span> stuckCount++;<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span> LOG.warn("Worker stuck " + worker +<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span> " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1985"></a>
-<span class="sourceLineNo">1986</span> }<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span> return stuckCount;<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span> }<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span><a name="line.1989"></a>
-<span class="sourceLineNo">1990</span> private void checkThreadCount(final int stuckCount) {<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span> // nothing to do if there are no runnable tasks<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span> if (stuckCount < 1 || !scheduler.hasRunnables()) return;<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span><a name="line.1993"></a>
-<span class="sourceLineNo">1994</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span> // and every handler is active.<a name="line.1995"></a>
-<span class="sourceLineNo">1996</span> final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span> if (stuckPerc >= addWorkerStuckPercentage &&<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span> activeExecutorCount.get() == workerThreads.size()) {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span> final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span> workerThreads.add(worker);<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span> worker.start();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span> LOG.debug("Added new worker thread " + worker);<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span> }<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span> }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span><a name="line.2005"></a>
-<span class="sourceLineNo">2006</span> private void refreshConfig() {<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span> }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span><a name="line.2014"></a>
-<span class="sourceLineNo">2015</span> @Override<a name="line.2015"></a>
-<span class="sourceLineNo">2016</span> public int getTimeoutInterval() {<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span> return timeoutInterval;<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span> }<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span> }<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>}<a name="line.2020"></a>
+<span class="sourceLineNo">1907</span> // Log every two seconds; send interrupt too.<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span> if (i > 0 && (i % 8) == 0) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span> LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span> StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime) +<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span> "; sending interrupt");<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span> interrupt();<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span> }<a name="line.1913"></a>
+<span class="sourceLineNo">1914</span> }<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span> } catch (InterruptedException e) {<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span> LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span> }<a name="line.1917"></a>
+<span class="sourceLineNo">1918</span> }<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span> }<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span><a name="line.1920"></a>
+<span class="sourceLineNo">1921</span> // ==========================================================================<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span> // Inline Chores (executors internal chores)<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span> // ==========================================================================<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span> private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1924"></a>
+<span class="sourceLineNo">1925</span> private long timeout;<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span><a name="line.1926"></a>
+<span class="sourceLineNo">1927</span> public abstract int getTimeoutInterval();<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span><a name="line.1928"></a>
+<span class="sourceLineNo">1929</span> protected void refreshTimeout() {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span> this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span> }<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span> @Override<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span> public long getTimeout() {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span> return timeout;<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span> }<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span> }<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span><a name="line.1938"></a>
+<span class="sourceLineNo">1939</span> // ----------------------------------------------------------------------------<a name="line.1939"></a>
+<span class="sourceLineNo">1940</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span> // full set of procedures pending and completed to write a compacted<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span> // version of the log (in case is a log)?<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span> // will have the tracker saying everything is in the last log.<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span> // ----------------------------------------------------------------------------<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span><a name="line.1946"></a>
+<span class="sourceLineNo">1947</span> private final class WorkerMonitor extends InlineChore {<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span><a name="line.1951"></a>
+<span class="sourceLineNo">1952</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span><a name="line.1955"></a>
+<span class="sourceLineNo">1956</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span><a name="line.1959"></a>
+<span class="sourceLineNo">1960</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span><a name="line.1963"></a>
+<span class="sourceLineNo">1964</span> public WorkerMonitor() {<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span> refreshConfig();<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span> }<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span><a name="line.1967"></a>
+<span class="sourceLineNo">1968</span> @Override<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span> public void run() {<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span> final int stuckCount = checkForStuckWorkers();<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span> checkThreadCount(stuckCount);<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span><a name="line.1972"></a>
+<span class="sourceLineNo">1973</span> // refresh interval (poor man dynamic conf update)<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span> refreshConfig();<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span> }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span><a name="line.1976"></a>
+<span class="sourceLineNo">1977</span> private int checkForStuckWorkers() {<a name="line.1977"></a>
+<span class="sourceLineNo">1978</span> // check if any of the worker is stuck<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span> int stuckCount = 0;<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span> for (WorkerThread worker: workerThreads) {<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span> continue;<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span> }<a name="line.1983"></a>
+<span class="sourceLineNo">1984</span><a name="line.1984"></a>
+<span class="sourceLineNo">1985</span> // WARN the worker is stuck<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span> stuckCount++;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span> LOG.warn("Worker stuck " + worker +<a name="line.1987"></a>
+<span class="sourceLineNo">1988</span> " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span> }<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span> return stuckCount;<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span> }<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span><a name="line.1992"></a>
+<span class="sourceLineNo">1993</span> private void checkThreadCount(final int stuckCount) {<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span> // nothing to do if there are no runnable tasks<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span> if (stuckCount < 1 || !scheduler.hasRunnables()) return;<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span><a name="line.1996"></a>
+<span class="sourceLineNo">1997</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1997"></a>
+<span class="sourceLineNo">1998</span> // and every handler is active.<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span> final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span> if (stuckPerc >= addWorkerStuckPercentage &&<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span> activeExecutorCount.get() == workerThreads.size()) {<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span> final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span> workerThreads.add(worker);<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span> worker.start();<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span> LOG.debug("Added new worker thread " + worker);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span> }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span> }<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span><a name="line.2008"></a>
+<span class="sourceLineNo">2009</span> private void refreshConfig() {<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span> }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span> @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span> public int getTimeoutInterval() {<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span> return timeoutInterval;<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span> }<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span> }<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>}<a name="line.2023"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.StoppableThread.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.StoppableThread.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.StoppableThread.html
index 7271567..a08bfa4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.StoppableThread.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.StoppableThread.html
@@ -518,10 +518,10 @@
<span class="sourceLineNo">510</span> // We have numThreads executor + one timer thread used for timing out<a name="line.510"></a>
<span class="sourceLineNo">511</span> // procedures and triggering periodic procedures.<a name="line.511"></a>
<span class="sourceLineNo">512</span> this.corePoolSize = numThreads;<a name="line.512"></a>
-<span class="sourceLineNo">513</span> LOG.info("Starting ProcedureExecutor Worker threads (ProcExecWrkr)=" + corePoolSize);<a name="line.513"></a>
+<span class="sourceLineNo">513</span> LOG.info("Starting ProcedureExecutor Worker threads (ProcedureExecutorWorker)=" + corePoolSize);<a name="line.513"></a>
<span class="sourceLineNo">514</span><a name="line.514"></a>
<span class="sourceLineNo">515</span> // Create the Thread Group for the executors<a name="line.515"></a>
-<span class="sourceLineNo">516</span> threadGroup = new ThreadGroup("ProcExecThrdGrp");<a name="line.516"></a>
+<span class="sourceLineNo">516</span> threadGroup = new ThreadGroup("ProcedureExecutorWorkerGroup");<a name="line.516"></a>
<span class="sourceLineNo">517</span><a name="line.517"></a>
<span class="sourceLineNo">518</span> // Create the timeout executor<a name="line.518"></a>
<span class="sourceLineNo">519</span> timeoutExecutor = new TimeoutExecutorThread(threadGroup);<a name="line.519"></a>
@@ -600,7 +600,7 @@
<span class="sourceLineNo">592</span> try {<a name="line.592"></a>
<span class="sourceLineNo">593</span> threadGroup.destroy();<a name="line.593"></a>
<span class="sourceLineNo">594</span> } catch (IllegalThreadStateException e) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span> LOG.error("Thread group " + threadGroup + " contains running threads");<a name="line.595"></a>
+<span class="sourceLineNo">595</span> LOG.error("ThreadGroup " + threadGroup + " contains running threads; " + e.getMessage());<a name="line.595"></a>
<span class="sourceLineNo">596</span> threadGroup.list();<a name="line.596"></a>
<span class="sourceLineNo">597</span> } finally {<a name="line.597"></a>
<span class="sourceLineNo">598</span> threadGroup = null;<a name="line.598"></a>
@@ -1717,7 +1717,7 @@
<span class="sourceLineNo">1709</span> private Procedure activeProcedure;<a name="line.1709"></a>
<span class="sourceLineNo">1710</span><a name="line.1710"></a>
<span class="sourceLineNo">1711</span> public WorkerThread(final ThreadGroup group) {<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span> super(group, "ProcExecWrkr-" + workerId.incrementAndGet());<a name="line.1712"></a>
+<span class="sourceLineNo">1712</span> super(group, "ProcedureExecutorWorker-" + workerId.incrementAndGet());<a name="line.1712"></a>
<span class="sourceLineNo">1713</span> setDaemon(true);<a name="line.1713"></a>
<span class="sourceLineNo">1714</span> }<a name="line.1714"></a>
<span class="sourceLineNo">1715</span><a name="line.1715"></a>
@@ -1760,7 +1760,7 @@
<span class="sourceLineNo">1752</span> } catch (Throwable t) {<a name="line.1752"></a>
<span class="sourceLineNo">1753</span> LOG.warn("Worker terminating UNNATURALLY " + this.activeProcedure, t);<a name="line.1753"></a>
<span class="sourceLineNo">1754</span> } finally {<a name="line.1754"></a>
-<span class="sourceLineNo">1755</span> LOG.debug("Worker terminated.");<a name="line.1755"></a>
+<span class="sourceLineNo">1755</span> LOG.trace("Worker terminated.");<a name="line.1755"></a>
<span class="sourceLineNo">1756</span> }<a name="line.1756"></a>
<span class="sourceLineNo">1757</span> workerThreads.remove(this);<a name="line.1757"></a>
<span class="sourceLineNo">1758</span> }<a name="line.1758"></a>
@@ -1912,120 +1912,123 @@
<span class="sourceLineNo">1904</span> for (int i = 0; isAlive(); ++i) {<a name="line.1904"></a>
<span class="sourceLineNo">1905</span> sendStopSignal();<a name="line.1905"></a>
<span class="sourceLineNo">1906</span> join(250);<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span> if (i > 0 && (i % 8) == 0) {<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span> LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span> StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime));<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span> }<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span> }<a name="line.1911"></a>
-<span class="sourceLineNo">1912</span> } catch (InterruptedException e) {<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span> LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span> }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span> }<a name="line.1915"></a>
-<span class="sourceLineNo">1916</span> }<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span><a name="line.1917"></a>
-<span class="sourceLineNo">1918</span> // ==========================================================================<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span> // Inline Chores (executors internal chores)<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span> // ==========================================================================<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span> private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span> private long timeout;<a name="line.1922"></a>
-<span class="sourceLineNo">1923</span><a name="line.1923"></a>
-<span class="sourceLineNo">1924</span> public abstract int getTimeoutInterval();<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span><a name="line.1925"></a>
-<span class="sourceLineNo">1926</span> protected void refreshTimeout() {<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span> this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span> }<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span><a name="line.1929"></a>
-<span class="sourceLineNo">1930</span> @Override<a name="line.1930"></a>
-<span class="sourceLineNo">1931</span> public long getTimeout() {<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span> return timeout;<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span> }<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span> }<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span><a name="line.1935"></a>
-<span class="sourceLineNo">1936</span> // ----------------------------------------------------------------------------<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1937"></a>
-<span class="sourceLineNo">1938</span> // full set of procedures pending and completed to write a compacted<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span> // version of the log (in case is a log)?<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span> // will have the tracker saying everything is in the last log.<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span> // ----------------------------------------------------------------------------<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span><a name="line.1943"></a>
-<span class="sourceLineNo">1944</span> private final class WorkerMonitor extends InlineChore {<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span><a name="line.1948"></a>
-<span class="sourceLineNo">1949</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span><a name="line.1952"></a>
-<span class="sourceLineNo">1953</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span><a name="line.1956"></a>
-<span class="sourceLineNo">1957</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span><a name="line.1960"></a>
-<span class="sourceLineNo">1961</span> public WorkerMonitor() {<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span> refreshConfig();<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span> }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span><a name="line.1964"></a>
-<span class="sourceLineNo">1965</span> @Override<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span> public void run() {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span> final int stuckCount = checkForStuckWorkers();<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span> checkThreadCount(stuckCount);<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span><a name="line.1969"></a>
-<span class="sourceLineNo">1970</span> // refresh interval (poor man dynamic conf update)<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span> refreshConfig();<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span> }<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span><a name="line.1973"></a>
-<span class="sourceLineNo">1974</span> private int checkForStuckWorkers() {<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span> // check if any of the worker is stuck<a name="line.1975"></a>
-<span class="sourceLineNo">1976</span> int stuckCount = 0;<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span> for (WorkerThread worker: workerThreads) {<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span> continue;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span> }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span> // WARN the worker is stuck<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span> stuckCount++;<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span> LOG.warn("Worker stuck " + worker +<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span> " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1985"></a>
-<span class="sourceLineNo">1986</span> }<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span> return stuckCount;<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span> }<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span><a name="line.1989"></a>
-<span class="sourceLineNo">1990</span> private void checkThreadCount(final int stuckCount) {<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span> // nothing to do if there are no runnable tasks<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span> if (stuckCount < 1 || !scheduler.hasRunnables()) return;<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span><a name="line.1993"></a>
-<span class="sourceLineNo">1994</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span> // and every handler is active.<a name="line.1995"></a>
-<span class="sourceLineNo">1996</span> final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span> if (stuckPerc >= addWorkerStuckPercentage &&<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span> activeExecutorCount.get() == workerThreads.size()) {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span> final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span> workerThreads.add(worker);<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span> worker.start();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span> LOG.debug("Added new worker thread " + worker);<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span> }<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span> }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span><a name="line.2005"></a>
-<span class="sourceLineNo">2006</span> private void refreshConfig() {<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span> }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span><a name="line.2014"></a>
-<span class="sourceLineNo">2015</span> @Override<a name="line.2015"></a>
-<span class="sourceLineNo">2016</span> public int getTimeoutInterval() {<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span> return timeoutInterval;<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span> }<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span> }<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>}<a name="line.2020"></a>
+<span class="sourceLineNo">1907</span> // Log every two seconds; send interrupt too.<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span> if (i > 0 && (i % 8) == 0) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span> LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span> StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime) +<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span> "; sending interrupt");<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span> interrupt();<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span> }<a name="line.1913"></a>
+<span class="sourceLineNo">1914</span> }<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span> } catch (InterruptedException e) {<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span> LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span> }<a name="line.1917"></a>
+<span class="sourceLineNo">1918</span> }<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span> }<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span><a name="line.1920"></a>
+<span class="sourceLineNo">1921</span> // ==========================================================================<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span> // Inline Chores (executors internal chores)<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span> // ==========================================================================<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span> private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1924"></a>
+<span class="sourceLineNo">1925</span> private long timeout;<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span><a name="line.1926"></a>
+<span class="sourceLineNo">1927</span> public abstract int getTimeoutInterval();<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span><a name="line.1928"></a>
+<span class="sourceLineNo">1929</span> protected void refreshTimeout() {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span> this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span> }<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span> @Override<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span> public long getTimeout() {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span> return timeout;<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span> }<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span> }<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span><a name="line.1938"></a>
+<span class="sourceLineNo">1939</span> // ----------------------------------------------------------------------------<a name="line.1939"></a>
+<span class="sourceLineNo">1940</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span> // full set of procedures pending and completed to write a compacted<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span> // version of the log (in case is a log)?<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span> // will have the tracker saying everything is in the last log.<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span> // ----------------------------------------------------------------------------<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span><a name="line.1946"></a>
+<span class="sourceLineNo">1947</span> private final class WorkerMonitor extends InlineChore {<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span><a name="line.1951"></a>
+<span class="sourceLineNo">1952</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span><a name="line.1955"></a>
+<span class="sourceLineNo">1956</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span><a name="line.1959"></a>
+<span class="sourceLineNo">1960</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span><a name="line.1963"></a>
+<span class="sourceLineNo">1964</span> public WorkerMonitor() {<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span> refreshConfig();<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span> }<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span><a name="line.1967"></a>
+<span class="sourceLineNo">1968</span> @Override<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span> public void run() {<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span> final int stuckCount = checkForStuckWorkers();<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span> checkThreadCount(stuckCount);<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span><a name="line.1972"></a>
+<span class="sourceLineNo">1973</span> // refresh interval (poor man dynamic conf update)<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span> refreshConfig();<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span> }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span><a name="line.1976"></a>
+<span class="sourceLineNo">1977</span> private int checkForStuckWorkers() {<a name="line.1977"></a>
+<span class="sourceLineNo">1978</span> // check if any of the worker is stuck<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span> int stuckCount = 0;<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span> for (WorkerThread worker: workerThreads) {<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span> continue;<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span> }<a name="line.1983"></a>
+<span class="sourceLineNo">1984</span><a name="line.1984"></a>
+<span class="sourceLineNo">1985</span> // WARN the worker is stuck<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span> stuckCount++;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span> LOG.warn("Worker stuck " + worker +<a name="line.1987"></a>
+<span class="sourceLineNo">1988</span> " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span> }<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span> return stuckCount;<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span> }<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span><a name="line.1992"></a>
+<span class="sourceLineNo">1993</span> private void checkThreadCount(final int stuckCount) {<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span> // nothing to do if there are no runnable tasks<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span> if (stuckCount < 1 || !scheduler.hasRunnables()) return;<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span><a name="line.1996"></a>
+<span class="sourceLineNo">1997</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1997"></a>
+<span class="sourceLineNo">1998</span> // and every handler is active.<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span> final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span> if (stuckPerc >= addWorkerStuckPercentage &&<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span> activeExecutorCount.get() == workerThreads.size()) {<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span> final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span> workerThreads.add(worker);<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span> worker.start();<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span> LOG.debug("Added new worker thread " + worker);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span> }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span> }<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span><a name="line.2008"></a>
+<span class="sourceLineNo">2009</span> private void refreshConfig() {<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span> }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span> @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span> public int getTimeoutInterval() {<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span> return timeoutInterval;<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span> }<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span> }<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>}<a name="line.2023"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/90170c41/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
----------------------------------------------------------------------
diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
index 7271567..a08bfa4 100644
--- a/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
+++ b/devapidocs/src-html/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.Testing.html
@@ -518,10 +518,10 @@
<span class="sourceLineNo">510</span> // We have numThreads executor + one timer thread used for timing out<a name="line.510"></a>
<span class="sourceLineNo">511</span> // procedures and triggering periodic procedures.<a name="line.511"></a>
<span class="sourceLineNo">512</span> this.corePoolSize = numThreads;<a name="line.512"></a>
-<span class="sourceLineNo">513</span> LOG.info("Starting ProcedureExecutor Worker threads (ProcExecWrkr)=" + corePoolSize);<a name="line.513"></a>
+<span class="sourceLineNo">513</span> LOG.info("Starting ProcedureExecutor Worker threads (ProcedureExecutorWorker)=" + corePoolSize);<a name="line.513"></a>
<span class="sourceLineNo">514</span><a name="line.514"></a>
<span class="sourceLineNo">515</span> // Create the Thread Group for the executors<a name="line.515"></a>
-<span class="sourceLineNo">516</span> threadGroup = new ThreadGroup("ProcExecThrdGrp");<a name="line.516"></a>
+<span class="sourceLineNo">516</span> threadGroup = new ThreadGroup("ProcedureExecutorWorkerGroup");<a name="line.516"></a>
<span class="sourceLineNo">517</span><a name="line.517"></a>
<span class="sourceLineNo">518</span> // Create the timeout executor<a name="line.518"></a>
<span class="sourceLineNo">519</span> timeoutExecutor = new TimeoutExecutorThread(threadGroup);<a name="line.519"></a>
@@ -600,7 +600,7 @@
<span class="sourceLineNo">592</span> try {<a name="line.592"></a>
<span class="sourceLineNo">593</span> threadGroup.destroy();<a name="line.593"></a>
<span class="sourceLineNo">594</span> } catch (IllegalThreadStateException e) {<a name="line.594"></a>
-<span class="sourceLineNo">595</span> LOG.error("Thread group " + threadGroup + " contains running threads");<a name="line.595"></a>
+<span class="sourceLineNo">595</span> LOG.error("ThreadGroup " + threadGroup + " contains running threads; " + e.getMessage());<a name="line.595"></a>
<span class="sourceLineNo">596</span> threadGroup.list();<a name="line.596"></a>
<span class="sourceLineNo">597</span> } finally {<a name="line.597"></a>
<span class="sourceLineNo">598</span> threadGroup = null;<a name="line.598"></a>
@@ -1717,7 +1717,7 @@
<span class="sourceLineNo">1709</span> private Procedure activeProcedure;<a name="line.1709"></a>
<span class="sourceLineNo">1710</span><a name="line.1710"></a>
<span class="sourceLineNo">1711</span> public WorkerThread(final ThreadGroup group) {<a name="line.1711"></a>
-<span class="sourceLineNo">1712</span> super(group, "ProcExecWrkr-" + workerId.incrementAndGet());<a name="line.1712"></a>
+<span class="sourceLineNo">1712</span> super(group, "ProcedureExecutorWorker-" + workerId.incrementAndGet());<a name="line.1712"></a>
<span class="sourceLineNo">1713</span> setDaemon(true);<a name="line.1713"></a>
<span class="sourceLineNo">1714</span> }<a name="line.1714"></a>
<span class="sourceLineNo">1715</span><a name="line.1715"></a>
@@ -1760,7 +1760,7 @@
<span class="sourceLineNo">1752</span> } catch (Throwable t) {<a name="line.1752"></a>
<span class="sourceLineNo">1753</span> LOG.warn("Worker terminating UNNATURALLY " + this.activeProcedure, t);<a name="line.1753"></a>
<span class="sourceLineNo">1754</span> } finally {<a name="line.1754"></a>
-<span class="sourceLineNo">1755</span> LOG.debug("Worker terminated.");<a name="line.1755"></a>
+<span class="sourceLineNo">1755</span> LOG.trace("Worker terminated.");<a name="line.1755"></a>
<span class="sourceLineNo">1756</span> }<a name="line.1756"></a>
<span class="sourceLineNo">1757</span> workerThreads.remove(this);<a name="line.1757"></a>
<span class="sourceLineNo">1758</span> }<a name="line.1758"></a>
@@ -1912,120 +1912,123 @@
<span class="sourceLineNo">1904</span> for (int i = 0; isAlive(); ++i) {<a name="line.1904"></a>
<span class="sourceLineNo">1905</span> sendStopSignal();<a name="line.1905"></a>
<span class="sourceLineNo">1906</span> join(250);<a name="line.1906"></a>
-<span class="sourceLineNo">1907</span> if (i > 0 && (i % 8) == 0) {<a name="line.1907"></a>
-<span class="sourceLineNo">1908</span> LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1908"></a>
-<span class="sourceLineNo">1909</span> StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime));<a name="line.1909"></a>
-<span class="sourceLineNo">1910</span> }<a name="line.1910"></a>
-<span class="sourceLineNo">1911</span> }<a name="line.1911"></a>
-<span class="sourceLineNo">1912</span> } catch (InterruptedException e) {<a name="line.1912"></a>
-<span class="sourceLineNo">1913</span> LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1913"></a>
-<span class="sourceLineNo">1914</span> }<a name="line.1914"></a>
-<span class="sourceLineNo">1915</span> }<a name="line.1915"></a>
-<span class="sourceLineNo">1916</span> }<a name="line.1916"></a>
-<span class="sourceLineNo">1917</span><a name="line.1917"></a>
-<span class="sourceLineNo">1918</span> // ==========================================================================<a name="line.1918"></a>
-<span class="sourceLineNo">1919</span> // Inline Chores (executors internal chores)<a name="line.1919"></a>
-<span class="sourceLineNo">1920</span> // ==========================================================================<a name="line.1920"></a>
-<span class="sourceLineNo">1921</span> private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1921"></a>
-<span class="sourceLineNo">1922</span> private long timeout;<a name="line.1922"></a>
-<span class="sourceLineNo">1923</span><a name="line.1923"></a>
-<span class="sourceLineNo">1924</span> public abstract int getTimeoutInterval();<a name="line.1924"></a>
-<span class="sourceLineNo">1925</span><a name="line.1925"></a>
-<span class="sourceLineNo">1926</span> protected void refreshTimeout() {<a name="line.1926"></a>
-<span class="sourceLineNo">1927</span> this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1927"></a>
-<span class="sourceLineNo">1928</span> }<a name="line.1928"></a>
-<span class="sourceLineNo">1929</span><a name="line.1929"></a>
-<span class="sourceLineNo">1930</span> @Override<a name="line.1930"></a>
-<span class="sourceLineNo">1931</span> public long getTimeout() {<a name="line.1931"></a>
-<span class="sourceLineNo">1932</span> return timeout;<a name="line.1932"></a>
-<span class="sourceLineNo">1933</span> }<a name="line.1933"></a>
-<span class="sourceLineNo">1934</span> }<a name="line.1934"></a>
-<span class="sourceLineNo">1935</span><a name="line.1935"></a>
-<span class="sourceLineNo">1936</span> // ----------------------------------------------------------------------------<a name="line.1936"></a>
-<span class="sourceLineNo">1937</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1937"></a>
-<span class="sourceLineNo">1938</span> // full set of procedures pending and completed to write a compacted<a name="line.1938"></a>
-<span class="sourceLineNo">1939</span> // version of the log (in case is a log)?<a name="line.1939"></a>
-<span class="sourceLineNo">1940</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.1940"></a>
-<span class="sourceLineNo">1941</span> // will have the tracker saying everything is in the last log.<a name="line.1941"></a>
-<span class="sourceLineNo">1942</span> // ----------------------------------------------------------------------------<a name="line.1942"></a>
-<span class="sourceLineNo">1943</span><a name="line.1943"></a>
-<span class="sourceLineNo">1944</span> private final class WorkerMonitor extends InlineChore {<a name="line.1944"></a>
-<span class="sourceLineNo">1945</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1945"></a>
-<span class="sourceLineNo">1946</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.1946"></a>
-<span class="sourceLineNo">1947</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1947"></a>
-<span class="sourceLineNo">1948</span><a name="line.1948"></a>
-<span class="sourceLineNo">1949</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1949"></a>
-<span class="sourceLineNo">1950</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1950"></a>
-<span class="sourceLineNo">1951</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1951"></a>
-<span class="sourceLineNo">1952</span><a name="line.1952"></a>
-<span class="sourceLineNo">1953</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1953"></a>
-<span class="sourceLineNo">1954</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.1954"></a>
-<span class="sourceLineNo">1955</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1955"></a>
-<span class="sourceLineNo">1956</span><a name="line.1956"></a>
-<span class="sourceLineNo">1957</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1957"></a>
-<span class="sourceLineNo">1958</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1958"></a>
-<span class="sourceLineNo">1959</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1959"></a>
-<span class="sourceLineNo">1960</span><a name="line.1960"></a>
-<span class="sourceLineNo">1961</span> public WorkerMonitor() {<a name="line.1961"></a>
-<span class="sourceLineNo">1962</span> refreshConfig();<a name="line.1962"></a>
-<span class="sourceLineNo">1963</span> }<a name="line.1963"></a>
-<span class="sourceLineNo">1964</span><a name="line.1964"></a>
-<span class="sourceLineNo">1965</span> @Override<a name="line.1965"></a>
-<span class="sourceLineNo">1966</span> public void run() {<a name="line.1966"></a>
-<span class="sourceLineNo">1967</span> final int stuckCount = checkForStuckWorkers();<a name="line.1967"></a>
-<span class="sourceLineNo">1968</span> checkThreadCount(stuckCount);<a name="line.1968"></a>
-<span class="sourceLineNo">1969</span><a name="line.1969"></a>
-<span class="sourceLineNo">1970</span> // refresh interval (poor man dynamic conf update)<a name="line.1970"></a>
-<span class="sourceLineNo">1971</span> refreshConfig();<a name="line.1971"></a>
-<span class="sourceLineNo">1972</span> }<a name="line.1972"></a>
-<span class="sourceLineNo">1973</span><a name="line.1973"></a>
-<span class="sourceLineNo">1974</span> private int checkForStuckWorkers() {<a name="line.1974"></a>
-<span class="sourceLineNo">1975</span> // check if any of the worker is stuck<a name="line.1975"></a>
-<span class="sourceLineNo">1976</span> int stuckCount = 0;<a name="line.1976"></a>
-<span class="sourceLineNo">1977</span> for (WorkerThread worker: workerThreads) {<a name="line.1977"></a>
-<span class="sourceLineNo">1978</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.1978"></a>
-<span class="sourceLineNo">1979</span> continue;<a name="line.1979"></a>
-<span class="sourceLineNo">1980</span> }<a name="line.1980"></a>
-<span class="sourceLineNo">1981</span><a name="line.1981"></a>
-<span class="sourceLineNo">1982</span> // WARN the worker is stuck<a name="line.1982"></a>
-<span class="sourceLineNo">1983</span> stuckCount++;<a name="line.1983"></a>
-<span class="sourceLineNo">1984</span> LOG.warn("Worker stuck " + worker +<a name="line.1984"></a>
-<span class="sourceLineNo">1985</span> " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1985"></a>
-<span class="sourceLineNo">1986</span> }<a name="line.1986"></a>
-<span class="sourceLineNo">1987</span> return stuckCount;<a name="line.1987"></a>
-<span class="sourceLineNo">1988</span> }<a name="line.1988"></a>
-<span class="sourceLineNo">1989</span><a name="line.1989"></a>
-<span class="sourceLineNo">1990</span> private void checkThreadCount(final int stuckCount) {<a name="line.1990"></a>
-<span class="sourceLineNo">1991</span> // nothing to do if there are no runnable tasks<a name="line.1991"></a>
-<span class="sourceLineNo">1992</span> if (stuckCount < 1 || !scheduler.hasRunnables()) return;<a name="line.1992"></a>
-<span class="sourceLineNo">1993</span><a name="line.1993"></a>
-<span class="sourceLineNo">1994</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1994"></a>
-<span class="sourceLineNo">1995</span> // and every handler is active.<a name="line.1995"></a>
-<span class="sourceLineNo">1996</span> final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1996"></a>
-<span class="sourceLineNo">1997</span> if (stuckPerc >= addWorkerStuckPercentage &&<a name="line.1997"></a>
-<span class="sourceLineNo">1998</span> activeExecutorCount.get() == workerThreads.size()) {<a name="line.1998"></a>
-<span class="sourceLineNo">1999</span> final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.1999"></a>
-<span class="sourceLineNo">2000</span> workerThreads.add(worker);<a name="line.2000"></a>
-<span class="sourceLineNo">2001</span> worker.start();<a name="line.2001"></a>
-<span class="sourceLineNo">2002</span> LOG.debug("Added new worker thread " + worker);<a name="line.2002"></a>
-<span class="sourceLineNo">2003</span> }<a name="line.2003"></a>
-<span class="sourceLineNo">2004</span> }<a name="line.2004"></a>
-<span class="sourceLineNo">2005</span><a name="line.2005"></a>
-<span class="sourceLineNo">2006</span> private void refreshConfig() {<a name="line.2006"></a>
-<span class="sourceLineNo">2007</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2007"></a>
-<span class="sourceLineNo">2008</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2008"></a>
-<span class="sourceLineNo">2009</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2009"></a>
-<span class="sourceLineNo">2010</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2010"></a>
-<span class="sourceLineNo">2011</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2011"></a>
-<span class="sourceLineNo">2012</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2012"></a>
-<span class="sourceLineNo">2013</span> }<a name="line.2013"></a>
-<span class="sourceLineNo">2014</span><a name="line.2014"></a>
-<span class="sourceLineNo">2015</span> @Override<a name="line.2015"></a>
-<span class="sourceLineNo">2016</span> public int getTimeoutInterval() {<a name="line.2016"></a>
-<span class="sourceLineNo">2017</span> return timeoutInterval;<a name="line.2017"></a>
-<span class="sourceLineNo">2018</span> }<a name="line.2018"></a>
-<span class="sourceLineNo">2019</span> }<a name="line.2019"></a>
-<span class="sourceLineNo">2020</span>}<a name="line.2020"></a>
+<span class="sourceLineNo">1907</span> // Log every two seconds; send interrupt too.<a name="line.1907"></a>
+<span class="sourceLineNo">1908</span> if (i > 0 && (i % 8) == 0) {<a name="line.1908"></a>
+<span class="sourceLineNo">1909</span> LOG.warn("Waiting termination of thread " + getName() + ", " +<a name="line.1909"></a>
+<span class="sourceLineNo">1910</span> StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime) +<a name="line.1910"></a>
+<span class="sourceLineNo">1911</span> "; sending interrupt");<a name="line.1911"></a>
+<span class="sourceLineNo">1912</span> interrupt();<a name="line.1912"></a>
+<span class="sourceLineNo">1913</span> }<a name="line.1913"></a>
+<span class="sourceLineNo">1914</span> }<a name="line.1914"></a>
+<span class="sourceLineNo">1915</span> } catch (InterruptedException e) {<a name="line.1915"></a>
+<span class="sourceLineNo">1916</span> LOG.warn(getName() + " join wait got interrupted", e);<a name="line.1916"></a>
+<span class="sourceLineNo">1917</span> }<a name="line.1917"></a>
+<span class="sourceLineNo">1918</span> }<a name="line.1918"></a>
+<span class="sourceLineNo">1919</span> }<a name="line.1919"></a>
+<span class="sourceLineNo">1920</span><a name="line.1920"></a>
+<span class="sourceLineNo">1921</span> // ==========================================================================<a name="line.1921"></a>
+<span class="sourceLineNo">1922</span> // Inline Chores (executors internal chores)<a name="line.1922"></a>
+<span class="sourceLineNo">1923</span> // ==========================================================================<a name="line.1923"></a>
+<span class="sourceLineNo">1924</span> private static abstract class InlineChore extends DelayedUtil.DelayedObject implements Runnable {<a name="line.1924"></a>
+<span class="sourceLineNo">1925</span> private long timeout;<a name="line.1925"></a>
+<span class="sourceLineNo">1926</span><a name="line.1926"></a>
+<span class="sourceLineNo">1927</span> public abstract int getTimeoutInterval();<a name="line.1927"></a>
+<span class="sourceLineNo">1928</span><a name="line.1928"></a>
+<span class="sourceLineNo">1929</span> protected void refreshTimeout() {<a name="line.1929"></a>
+<span class="sourceLineNo">1930</span> this.timeout = EnvironmentEdgeManager.currentTime() + getTimeoutInterval();<a name="line.1930"></a>
+<span class="sourceLineNo">1931</span> }<a name="line.1931"></a>
+<span class="sourceLineNo">1932</span><a name="line.1932"></a>
+<span class="sourceLineNo">1933</span> @Override<a name="line.1933"></a>
+<span class="sourceLineNo">1934</span> public long getTimeout() {<a name="line.1934"></a>
+<span class="sourceLineNo">1935</span> return timeout;<a name="line.1935"></a>
+<span class="sourceLineNo">1936</span> }<a name="line.1936"></a>
+<span class="sourceLineNo">1937</span> }<a name="line.1937"></a>
+<span class="sourceLineNo">1938</span><a name="line.1938"></a>
+<span class="sourceLineNo">1939</span> // ----------------------------------------------------------------------------<a name="line.1939"></a>
+<span class="sourceLineNo">1940</span> // TODO-MAYBE: Should we provide a InlineChore to notify the store with the<a name="line.1940"></a>
+<span class="sourceLineNo">1941</span> // full set of procedures pending and completed to write a compacted<a name="line.1941"></a>
+<span class="sourceLineNo">1942</span> // version of the log (in case is a log)?<a name="line.1942"></a>
+<span class="sourceLineNo">1943</span> // In theory no, procedures are have a short life, so at some point the store<a name="line.1943"></a>
+<span class="sourceLineNo">1944</span> // will have the tracker saying everything is in the last log.<a name="line.1944"></a>
+<span class="sourceLineNo">1945</span> // ----------------------------------------------------------------------------<a name="line.1945"></a>
+<span class="sourceLineNo">1946</span><a name="line.1946"></a>
+<span class="sourceLineNo">1947</span> private final class WorkerMonitor extends InlineChore {<a name="line.1947"></a>
+<span class="sourceLineNo">1948</span> public static final String WORKER_MONITOR_INTERVAL_CONF_KEY =<a name="line.1948"></a>
+<span class="sourceLineNo">1949</span> "hbase.procedure.worker.monitor.interval.msec";<a name="line.1949"></a>
+<span class="sourceLineNo">1950</span> private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 5000; // 5sec<a name="line.1950"></a>
+<span class="sourceLineNo">1951</span><a name="line.1951"></a>
+<span class="sourceLineNo">1952</span> public static final String WORKER_STUCK_THRESHOLD_CONF_KEY =<a name="line.1952"></a>
+<span class="sourceLineNo">1953</span> "hbase.procedure.worker.stuck.threshold.msec";<a name="line.1953"></a>
+<span class="sourceLineNo">1954</span> private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 10000; // 10sec<a name="line.1954"></a>
+<span class="sourceLineNo">1955</span><a name="line.1955"></a>
+<span class="sourceLineNo">1956</span> public static final String WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY =<a name="line.1956"></a>
+<span class="sourceLineNo">1957</span> "hbase.procedure.worker.add.stuck.percentage";<a name="line.1957"></a>
+<span class="sourceLineNo">1958</span> private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck<a name="line.1958"></a>
+<span class="sourceLineNo">1959</span><a name="line.1959"></a>
+<span class="sourceLineNo">1960</span> private float addWorkerStuckPercentage = DEFAULT_WORKER_ADD_STUCK_PERCENTAGE;<a name="line.1960"></a>
+<span class="sourceLineNo">1961</span> private int timeoutInterval = DEFAULT_WORKER_MONITOR_INTERVAL;<a name="line.1961"></a>
+<span class="sourceLineNo">1962</span> private int stuckThreshold = DEFAULT_WORKER_STUCK_THRESHOLD;<a name="line.1962"></a>
+<span class="sourceLineNo">1963</span><a name="line.1963"></a>
+<span class="sourceLineNo">1964</span> public WorkerMonitor() {<a name="line.1964"></a>
+<span class="sourceLineNo">1965</span> refreshConfig();<a name="line.1965"></a>
+<span class="sourceLineNo">1966</span> }<a name="line.1966"></a>
+<span class="sourceLineNo">1967</span><a name="line.1967"></a>
+<span class="sourceLineNo">1968</span> @Override<a name="line.1968"></a>
+<span class="sourceLineNo">1969</span> public void run() {<a name="line.1969"></a>
+<span class="sourceLineNo">1970</span> final int stuckCount = checkForStuckWorkers();<a name="line.1970"></a>
+<span class="sourceLineNo">1971</span> checkThreadCount(stuckCount);<a name="line.1971"></a>
+<span class="sourceLineNo">1972</span><a name="line.1972"></a>
+<span class="sourceLineNo">1973</span> // refresh interval (poor man dynamic conf update)<a name="line.1973"></a>
+<span class="sourceLineNo">1974</span> refreshConfig();<a name="line.1974"></a>
+<span class="sourceLineNo">1975</span> }<a name="line.1975"></a>
+<span class="sourceLineNo">1976</span><a name="line.1976"></a>
+<span class="sourceLineNo">1977</span> private int checkForStuckWorkers() {<a name="line.1977"></a>
+<span class="sourceLineNo">1978</span> // check if any of the worker is stuck<a name="line.1978"></a>
+<span class="sourceLineNo">1979</span> int stuckCount = 0;<a name="line.1979"></a>
+<span class="sourceLineNo">1980</span> for (WorkerThread worker: workerThreads) {<a name="line.1980"></a>
+<span class="sourceLineNo">1981</span> if (worker.getCurrentRunTime() < stuckThreshold) {<a name="line.1981"></a>
+<span class="sourceLineNo">1982</span> continue;<a name="line.1982"></a>
+<span class="sourceLineNo">1983</span> }<a name="line.1983"></a>
+<span class="sourceLineNo">1984</span><a name="line.1984"></a>
+<span class="sourceLineNo">1985</span> // WARN the worker is stuck<a name="line.1985"></a>
+<span class="sourceLineNo">1986</span> stuckCount++;<a name="line.1986"></a>
+<span class="sourceLineNo">1987</span> LOG.warn("Worker stuck " + worker +<a name="line.1987"></a>
+<span class="sourceLineNo">1988</span> " run time " + StringUtils.humanTimeDiff(worker.getCurrentRunTime()));<a name="line.1988"></a>
+<span class="sourceLineNo">1989</span> }<a name="line.1989"></a>
+<span class="sourceLineNo">1990</span> return stuckCount;<a name="line.1990"></a>
+<span class="sourceLineNo">1991</span> }<a name="line.1991"></a>
+<span class="sourceLineNo">1992</span><a name="line.1992"></a>
+<span class="sourceLineNo">1993</span> private void checkThreadCount(final int stuckCount) {<a name="line.1993"></a>
+<span class="sourceLineNo">1994</span> // nothing to do if there are no runnable tasks<a name="line.1994"></a>
+<span class="sourceLineNo">1995</span> if (stuckCount < 1 || !scheduler.hasRunnables()) return;<a name="line.1995"></a>
+<span class="sourceLineNo">1996</span><a name="line.1996"></a>
+<span class="sourceLineNo">1997</span> // add a new thread if the worker stuck percentage exceed the threshold limit<a name="line.1997"></a>
+<span class="sourceLineNo">1998</span> // and every handler is active.<a name="line.1998"></a>
+<span class="sourceLineNo">1999</span> final float stuckPerc = ((float)stuckCount) / workerThreads.size();<a name="line.1999"></a>
+<span class="sourceLineNo">2000</span> if (stuckPerc >= addWorkerStuckPercentage &&<a name="line.2000"></a>
+<span class="sourceLineNo">2001</span> activeExecutorCount.get() == workerThreads.size()) {<a name="line.2001"></a>
+<span class="sourceLineNo">2002</span> final WorkerThread worker = new WorkerThread(threadGroup);<a name="line.2002"></a>
+<span class="sourceLineNo">2003</span> workerThreads.add(worker);<a name="line.2003"></a>
+<span class="sourceLineNo">2004</span> worker.start();<a name="line.2004"></a>
+<span class="sourceLineNo">2005</span> LOG.debug("Added new worker thread " + worker);<a name="line.2005"></a>
+<span class="sourceLineNo">2006</span> }<a name="line.2006"></a>
+<span class="sourceLineNo">2007</span> }<a name="line.2007"></a>
+<span class="sourceLineNo">2008</span><a name="line.2008"></a>
+<span class="sourceLineNo">2009</span> private void refreshConfig() {<a name="line.2009"></a>
+<span class="sourceLineNo">2010</span> addWorkerStuckPercentage = conf.getFloat(WORKER_ADD_STUCK_PERCENTAGE_CONF_KEY,<a name="line.2010"></a>
+<span class="sourceLineNo">2011</span> DEFAULT_WORKER_ADD_STUCK_PERCENTAGE);<a name="line.2011"></a>
+<span class="sourceLineNo">2012</span> timeoutInterval = conf.getInt(WORKER_MONITOR_INTERVAL_CONF_KEY,<a name="line.2012"></a>
+<span class="sourceLineNo">2013</span> DEFAULT_WORKER_MONITOR_INTERVAL);<a name="line.2013"></a>
+<span class="sourceLineNo">2014</span> stuckThreshold = conf.getInt(WORKER_STUCK_THRESHOLD_CONF_KEY,<a name="line.2014"></a>
+<span class="sourceLineNo">2015</span> DEFAULT_WORKER_STUCK_THRESHOLD);<a name="line.2015"></a>
+<span class="sourceLineNo">2016</span> }<a name="line.2016"></a>
+<span class="sourceLineNo">2017</span><a name="line.2017"></a>
+<span class="sourceLineNo">2018</span> @Override<a name="line.2018"></a>
+<span class="sourceLineNo">2019</span> public int getTimeoutInterval() {<a name="line.2019"></a>
+<span class="sourceLineNo">2020</span> return timeoutInterval;<a name="line.2020"></a>
+<span class="sourceLineNo">2021</span> }<a name="line.2021"></a>
+<span class="sourceLineNo">2022</span> }<a name="line.2022"></a>
+<span class="sourceLineNo">2023</span>}<a name="line.2023"></a>